Softplus and more (#65) (#66)

* Initial commit

* A little nicer

* Fix the docs

* Add rsqrt

* Update CHANGELOG

Author: andrewdalpino

CHANGELOG.md

@@ -1,5 +1,10 @@
 # Change Log
 
+- 4.1.0
+    - Added `softplus()` and `erf()` special functions
+    - Added reciprocal sqrt `rsqrt()` operation to unary interface
+    - Added `sinh()` and `cosh()` trigonometric functions
+
 - 4.0.0
     - Added `fromArray()` factory method to build from PHP array
     - `build()` and `quick()` factory methods are now deprecated

benchmarks/Functions/SigmoidMatrixBench.php

@@ -0,0 +1,45 @@
+<?php
+
+namespace Tensor\Benchmarks\Functions;
+
+use Tensor\Matrix;
+
+/**
+ * @Groups({"Functions"})
+ * @BeforeMethods({"setUp"})
+ */
+class SigmoidMatrixBench
+{
+    /**
+     * @var Matrix
+     */
+    protected $a;
+
+    public function setUp() : void
+    {
+        $this->a = Matrix::uniform(500, 500);
+    }
+
+    /**
+     * @Subject
+     * @Iterations(5)
+     * @OutputTimeUnit("seconds", precision=3)
+     */
+    public function chained() : void
+    {
+        $this->a->negate()
+            ->exp()
+            ->addScalar(1.0)
+            ->reciprocal();
+    }
+
+    /**
+     * @Subject
+     * @Iterations(5)
+     * @OutputTimeUnit("seconds", precision=3)
+     */
+    public function map() : void
+    {
+        $this->a->sigmoid();
+    }
+}

composer.json

@@ -38,7 +38,8 @@
             "Tensor\\": "src/"
         },
         "files": [
-            "src/constants.php"
+            "src/constants.php",
+            "src/math.php"
         ]
     },
     "autoload-dev": {

docs/Matrix.md

@@ -414,6 +414,7 @@ See [Unary](interfaces/unary.md). Each method returns a new `Matrix`.
 - `square() : self` — square of the matrix element-wise
 - `sqrt() : self` — square root of the matrix
 - `reciprocal() : self` — element-wise reciprocal of the matrix
+- `rsqrt() : self` — element-wise reciprocal square root of the matrix, i.e. `1 / sqrt(x)`
 - `exp() : self` — exponential of the matrix
 - `expm1() : self` — exponential of each element minus 1
 - `log(float $base = M_E) : self` — logarithm of the matrix in the specified base
@@ -423,7 +424,6 @@ See [Unary](interfaces/unary.md). Each method returns a new `Matrix`.
 - `ceil() : self` — round up to the nearest integer
 - `sign() : self` — element-wise sign indication (`1.0`, `-1.0`, `0.0`)
 - `negate() : self` — take the negative of each value element-wise
-- `sigmoid() : self` — element-wise sigmoid of the matrix
 
 ## Trigonometric
 
@@ -435,6 +435,9 @@ See [Trigonometric](interfaces/trigonometric.md). Each method returns a new `Mat
 - `acos()` — arc cosine of the matrix
 - `tan()` — tangent of the matrix
 - `atan()` — arc tangent of the matrix
+- `tanh()` — hyperbolic tangent of the matrix
+- `cosh()` — hyperbolic cosine of the matrix
+- `sinh()` — hyperbolic sine of the matrix
 - `rad2deg()` — convert angles from radians to degrees
 - `deg2rad()` — convert angles from degrees to radians
 
@@ -456,7 +459,10 @@ See [Reductions](interfaces/reductions.md). For a `Matrix`, row-wise reductions
 
 See [Special](interfaces/special.md).
 
+- `sigmoid() : self` — the element-wise sigmoid of the matrix, i.e. `1 / (1 + exp(-x))`; always in `(0, 1)`
+- `softplus() : self` — the element-wise softplus of the matrix: `log(1 + e^{x})`; a smooth approximation of `max(0, x)` with derivative equal to the sigmoid
 - `softmax() : self` — the softmax of the matrix; each row is normalized independently so the elements of every row sum to `1.0` and the shape is preserved
+- `erf() : self` — the element-wise error function of the matrix; `erf(-x) = -erf(x)` and `erf(±5.0) ≈ ±1.0`
 
 ## Clipping
 

docs/Vector.md

@@ -314,6 +314,7 @@ See [Unary](interfaces/unary.md). Each method returns a new `Vector`.
 - `square() : self` — square the vector
 - `sqrt() : self` — square root of the vector
 - `reciprocal() : self` — element-wise reciprocal of the vector
+- `rsqrt() : self` — element-wise reciprocal square root of the vector, i.e. `1 / sqrt(x)`
 - `exp() : self` — exponentiate each element
 - `expm1() : self` — exponential of each element minus 1
 - `log(float $base = M_E) : self` — log to the given base of each element
@@ -323,7 +324,6 @@ See [Unary](interfaces/unary.md). Each method returns a new `Vector`.
 - `ceil() : self` — round up to the nearest integer
 - `sign() : self` — element-wise sign indication (`1.0`, `-1.0`, `0.0`)
 - `negate() : self` — negate each value
-- `sigmoid() : self` — element-wise sigmoid of the vector
 
 ## Trigonometric
 
@@ -336,6 +336,8 @@ See [Trigonometric](interfaces/trigonometric.md). Each method returns a new `Vec
 - `tan()` — tangent of the vector
 - `atan()` — arc tangent of the vector
 - `tanh()` — hyperbolic tangent of the vector
+- `cosh()` — hyperbolic cosine of the vector
+- `sinh()` — hyperbolic sine of the vector
 - `rad2deg()` — convert angles from radians to degrees
 - `deg2rad()` — convert angles from degrees to radians
 
@@ -356,7 +358,10 @@ See [Reductions](interfaces/reductions.md). For a `Vector` these return scalar `
 
 See [Special](interfaces/special.md).
 
+- `sigmoid() : self` — the element-wise sigmoid of the vector, i.e. `1 / (1 + exp(-x))`; always in `(0, 1)`
+- `softplus() : self` — the element-wise softplus of the vector: `log(1 + e^{x})`; a smooth approximation of `max(0, x)` with derivative equal to the sigmoid
 - `softmax() : self` — the softmax of the vector; the whole vector is treated as a single row so the elements sum to `1.0`
+- `erf() : self` — the element-wise error function of the vector; `erf(-x) = -erf(x)` and `erf(±5.0) ≈ ±1.0`
 
 ## Clipping
 

docs/interfaces/special.md

@@ -6,14 +6,26 @@ Special tensor functions.
 
 ## Overview
 
-`Special` defines higher-level functions that combine a transform with a normalization across more than one element.
+`Special` defines higher-order transcendental functions — element-wise transforms (`sigmoid`, `softplus`) and row-wise normalizations (`softmax`).
 
 ```php
 interface Special
 ```
 
 ## Methods
 
+### `sigmoid() : mixed`
+
+Return the element-wise sigmoid of the tensor, i.e. `1 / (1 + exp(-x))`.
+
+The result is always in the open interval `(0, 1)` and saturates to `1.0` and `0.0` for large positive and negative inputs respectively.
+
+### `softplus() : mixed`
+
+Return the element-wise softplus of the tensor: `log(1 + exp(x))`.
+
+A smooth approximation of `max(0, x)` whose derivative is the sigmoid. Computed as `x < 0: log(1 + exp(x))` and `x >= 0: x + log(1 + exp(-x))` so the exponentially large argument never overflows.
+
 ### `softmax() : mixed`
 
 Return the softmax of the tensor.
@@ -23,3 +35,9 @@ The maximum of each normalized row is subtracted before exponentiating, so the r
 - For `Matrix` each row is normalized independently, so the elements of every row sum to `1.0` and the shape is preserved. Normalization is across the columns *within* each row, never across rows.
 - For `Vector` the whole vector is a single row, so its elements sum to `1.0`.
 - For `ColumnVector` the whole column is a single row, so its elements sum to `1.0`.
+
+### `erf() : mixed`
+
+Return the element-wise error function of the tensor: `erf(x) = (2 / sqrt(pi)) * integral_0^x exp(-t^2) dt`.
+
+The result is in the range `[-1.0, 1.0]` and is an odd function: `erf(-x) = -erf(x)`. For `|x| >= 5.0` the value saturates to `sign(x)`.

docs/interfaces/tensor.md

@@ -20,10 +20,10 @@ interface Tensor extends ArrayLike, Arithmetic, Comparable,
 | [ArrayLike](arraylike.md) | `shape`, `shapeString`, `size`, `map`, `reduce`, `asArray` |
 | [Arithmetic](arithmetic.md) | `multiply`, `divide`, `add`, `subtract`, `pow`, `mod` |
 | [Comparable](comparable.md) | `equal`, `notEqual`, `greater`, `greaterEqual`, `less`, `lessEqual` |
-| [Unary](unary.md) | `abs`, `square`, `sqrt`, `reciprocal`, `exp`, `expm1`, `log`, `log1p`, `round`, `floor`, `ceil`, `clip`, `clipLower`, `clipUpper`, `sign`, `negate`, `sigmoid` |
-| [Trigonometric](trigonometric.md) | `sin`, `asin`, `cos`, `acos`, `tan`, `atan`, `tanh`, `rad2deg`, `deg2rad` |
+| [Unary](unary.md) | `abs`, `square`, `sqrt`, `reciprocal`, `rsqrt`, `exp`, `expm1`, `log`, `log1p`, `round`, `floor`, `ceil`, `clip`, `clipLower`, `clipUpper`, `sign`, `negate` |
+| [Trigonometric](trigonometric.md) | `sin`, `asin`, `cos`, `acos`, `tan`, `atan`, `tanh`, `cosh`, `sinh`, `rad2deg`, `deg2rad` |
 | [Reductions](reductions.md) | `sum`, `product`, `min`, `max`, `mean`, `variance`, `median`, `quantile` |
-| [Special](special.md) | `softmax` |
+| [Special](special.md) | `sigmoid`, `softplus`, `softmax`, `erf` |
 
 Additionally, because `ArrayLike` extends `ArrayAccess`, `IteratorAggregate`, and `Countable`, every tensor is array-accessible, iterable, and countable.
 

docs/interfaces/trigonometric.md

@@ -1,12 +1,12 @@
 # Trigonometric
 
-Element-wise trigonometric functions.
+Element-wise trigonometric and hyperbolic functions.
 
 - **Namespace:** `Tensor\Trigonometric`
 
 ## Overview
 
-`Trigonometric` defines element-wise trigonometric operations. Each method returns a new tensor of the same shape as `$this`. Angles are expressed in radians unless converting to/from degrees.
+`Trigonometric` defines element-wise trigonometric and hyperbolic operations. Each method returns a new tensor of the same shape as `$this`. Angles are expressed in radians unless converting to/from degrees.
 
 ```php
 interface Trigonometric
@@ -40,12 +40,20 @@ Compute the arc tangent of the tensor.
 
 ### `tanh() : mixed`
 
-Return the hyperbolic tangent of the tensor.
+Return the hyperbolic tangent of the tensor: `tanh(x) = sinh(x) / cosh(x)`.
+
+### `cosh() : mixed`
+
+Return the hyperbolic cosine of the tensor: `cosh(x) = (e^x + e^(-x)) / 2`.
+
+### `sinh() : mixed`
+
+Return the hyperbolic sine of the tensor: `sinh(x) = (e^x - e^(-x)) / 2`.
 
 ### `rad2deg() : mixed`
 
 Convert angles from radians to degrees.
 
 ### `deg2rad() : mixed`
 
-Convert angles from degrees to radians.
+Convert angles from degrees to radians.

docs/interfaces/unary.md

@@ -30,6 +30,10 @@ Return the square root of the tensor.
 
 Return the element-wise reciprocal of the tensor.
 
+### `rsqrt() : mixed`
+
+Return the reciprocal square root of the tensor, i.e. `1 / sqrt(x)`.
+
 ### `exp() : mixed`
 
 Return the exponential of the tensor.
@@ -84,10 +88,4 @@ Return the element-wise sign indication (`1.0`, `-1.0`, or `0.0`).
 
 ### `negate() : mixed`
 
-Negate the tensor, i.e. take the negative.
-
-### `sigmoid() : mixed`
-
-Return the element-wise sigmoid of the tensor, i.e. `1 / (1 + exp(-x))`.
-
-The result is always in the open interval `(0, 1)` and saturates to `1.0` and `0.0` for large positive and negative inputs respectively.
+Negate the tensor, i.e. take the negative.

src/Matrix.php

@@ -1409,6 +1409,16 @@ public function sqrt() : self
         return $this->map('sqrt');
     }
 
+    /**
+     * Return the reciprocal square root of the matrix.
+     *
+     * @return self
+     */
+    public function rsqrt() : self
+    {
+        return $this->map('\Tensor\rsqrt');
+    }
+
     /**
      * Return the exponential of the matrix.
      *
@@ -1532,6 +1542,26 @@ public function tanh() : self
         return $this->map('tanh');
     }
 
+    /**
+     * Return the hyperbolic cosine of the matrix.
+     *
+     * @return self
+     */
+    public function cosh() : self
+    {
+        return $this->map('cosh');
+    }
+
+    /**
+     * Return the hyperbolic sine of the matrix.
+     *
+     * @return self
+     */
+    public function sinh() : self
+    {
+        return $this->map('sinh');
+    }
+
     /**
      * Convert angles from radians to degrees.
      *
@@ -1929,10 +1959,7 @@ public function negate() : self
      */
     public function sigmoid() : self
     {
-        return $this->negate()
-            ->exp()
-            ->addScalar(1.0)
-            ->reciprocal();
+        return $this->map('\Tensor\sigmoid');
     }
 
     /**
@@ -1943,12 +1970,55 @@ public function sigmoid() : self
      */
     public function softmax() : self
     {
-        $a = $this->subtractColumnVector($this->max())
-            ->exp();
+        $a = $this->subtractColumnVector($this->max())->exp();
 
         return $a->divideColumnVector($a->sum());
     }
 
+    /**
+     * Return the element-wise error function of the matrix.
+     *
+     * @return self
+     */
+    public function erf() : self
+    {
+        $b = [];
+
+        foreach ($this->a as $rowA) {
+            $rowB = [];
+
+            foreach ($rowA as $valueA) {
+                $rowB[] = erf($valueA);
+            }
+
+            $b[] = $rowB;
+        }
+
+        return self::fromArray($b, false);
+    }
+
+    /**
+     * Return the element-wise softplus of the matrix.
+     *
+     * @return self
+     */
+    public function softplus() : self
+    {
+        $b = [];
+
+        foreach ($this->a as $rowA) {
+            $rowB = [];
+
+            foreach ($rowA as $valueA) {
+                $rowB[] = softplus($valueA);
+            }
+
+            $b[] = $rowB;
+        }
+
+        return self::fromArray($b, false);
+    }
+
     /**
      * Attach matrix b above this matrix.
      *