Convert activation functions to numpower

.github/workflows/ci.yml

@@ -8,14 +8,8 @@ jobs:
     runs-on: ${{ matrix.operating-system }}
     strategy:
       matrix:
-<<<<<<< HEAD
         operating-system: [windows-latest, ubuntu-latest, macos-latest]
         php-versions: ['8.4']
-=======
-        operating-system: [ubuntu-latest, macos-latest]
-        php-versions: ['8.0', '8.1', '8.2']
->>>>>>> master
-
     steps:
       - name: Checkout
         uses: actions/checkout@v3

.gitignore

@@ -13,3 +13,12 @@ pyvenv.cfg
 .idea
 .vscode
 .vs
+
+# Cache files
+/runtime/
+
+# Docker related files
+Dockerfile
+/docker/
+docker-compose.yml
+Makefile

.php-cs-fixer.dist.php

@@ -3,115 +3,128 @@
 use PhpCsFixer\Finder;
 use PhpCsFixer\Config;
 
-$finder = Finder::create()->in(__DIR__)
-    ->exclude('docs');
+$finder = Finder::create()
+    ->exclude([
+        __DIR__ . '/docs/',
+        __DIR__ . '/vendor/',
+    ])
+    ->in(__DIR__)
+    ->append([
+        __FILE__,
+    ]);
 
 $config = new Config();
+$config
+    ->setCacheFile(__DIR__ . '/runtime/.php-cs-fixer.cache')
+    ->setRules(
+        [
+            '@PSR2' => true,
+            '@PHP84Migration' => true,
+            'align_multiline_comment' => true,
+            'array_syntax' => ['syntax' => 'short'],
+            'backtick_to_shell_exec' => true,
+            'binary_operator_spaces' => true,
+            'blank_lines_before_namespace' => true,
+            'blank_line_after_namespace' => true,
+            'blank_line_after_opening_tag' => true,
+            'blank_line_before_statement' => [
+                'statements' => [
+                    'break', 'case', 'continue', 'declare', 'default', 'do', 'for',
+                    'if', 'foreach', 'return', 'switch', 'try', 'while',
+                ],
+            ],
+            'cast_spaces' => ['space' => 'single'],
+            'class_attributes_separation' => true,
+            'combine_consecutive_issets' => true,
+            'combine_consecutive_unsets' => true,
+            'compact_nullable_type_declaration' => true,
+            'concat_space' => ['spacing' => 'one'],
+            'fully_qualified_strict_types' => true,
+            'increment_style' => ['style' => 'pre'],
+            'linebreak_after_opening_tag' => true,
+            'list_syntax' => ['syntax' => 'short'],
+            'lowercase_cast' => true,
+            'lowercase_static_reference' => true,
+            'magic_constant_casing' => true,
+            'magic_method_casing' => true,
+            'multiline_comment_opening_closing' => true,
+            'multiline_whitespace_before_semicolons' => [
+                'strategy' => 'no_multi_line',
+            ],
+            'native_function_casing' => true,
+            'native_type_declaration_casing' => true,
+            'new_with_parentheses' => true,
+            'no_alternative_syntax' => true,
+            'no_blank_lines_after_class_opening' => true,
+            'no_blank_lines_after_phpdoc' => true,
+            'no_empty_statement' => true,
+            'no_extra_blank_lines' => true,
+            'no_leading_import_slash' => true,
+            'no_leading_namespace_whitespace' => true,
+            'no_mixed_echo_print' => ['use' => 'echo'],
+            'no_null_property_initialization' => true,
+            'no_short_bool_cast' => true,
+            'no_singleline_whitespace_before_semicolons' => true,
+            'no_spaces_around_offset' => true,
+            'no_superfluous_phpdoc_tags' => false,
+            'no_superfluous_elseif' => true,
+            'no_trailing_comma_in_singleline' => true,
+            'no_unneeded_control_parentheses' => true,
+            'no_unneeded_braces' => true,
+            'no_unset_cast' => true,
+            'no_unused_imports' => true,
+            'no_useless_else' => true,
+            'no_useless_return' => true,
+            'no_whitespace_before_comma_in_array' => true,
+            'no_whitespace_in_blank_line' => true,
+            'normalize_index_brace' => true,
+            'nullable_type_declaration_for_default_null_value' => true,
+            'object_operator_without_whitespace' => true,
+            'ordered_class_elements' => [
+                'order' => [
+                    'use_trait', 'constant_public', 'constant_protected',
+                    'constant_private', 'property_public_static', 'property_protected_static',
+                    'property_private_static', 'property_public', 'property_protected',
+                    'property_private', 'method_public_static', 'method_protected_static',
+                    'method_private_static', 'construct', 'destruct', 'phpunit',
+                    'method_public', 'method_protected', 'method_private', 'magic',
+                ],
+                'sort_algorithm' => 'none',
+            ],
+            'php_unit_fqcn_annotation' => true,
+            'php_unit_method_casing' => ['case' => 'camel_case'],
+            'phpdoc_add_missing_param_annotation' => ['only_untyped' => false],
+            'phpdoc_align' => ['align' => 'left'],
+            'phpdoc_line_span' => [
+                'const' => 'multi',
+                'method' => 'multi',
+                'property' => 'multi',
+            ],
+            'phpdoc_no_access' => true,
+            'phpdoc_no_empty_return' => true,
+            'phpdoc_no_useless_inheritdoc' => true,
+            'phpdoc_order' => true,
+            'phpdoc_scalar' => true,
+            'phpdoc_single_line_var_spacing' => true,
+            'phpdoc_to_comment' => false,
+            'phpdoc_trim' => true,
+            'phpdoc_trim_consecutive_blank_line_separation' => true,
+            'phpdoc_var_without_name' => true,
+            'protected_to_private' => true,
+            'return_assignment' => false,
+            'return_type_declaration' => ['space_before' => 'one'],
+            'semicolon_after_instruction' => true,
+            'short_scalar_cast' => true,
+            'simplified_null_return' => true,
+            'single_quote' => true,
+            'single_line_comment_style' => true,
+            'ternary_operator_spaces' => true,
+            'ternary_to_null_coalescing' => true,
+            'type_declaration_spaces' => true,
+            'trim_array_spaces' => true,
+            'unary_operator_spaces' => true,
+            'whitespace_after_comma_in_array' => true,
+        ]
+    )->setFinder($finder);
 
-return $config->setRules([
-    '@PSR2' => true,
-    'align_multiline_comment' => true,
-    'array_syntax' => ['syntax' => 'short'],
-    'backtick_to_shell_exec' => true,
-    'binary_operator_spaces' => true,
-    'blank_lines_before_namespace' => true,
-    'blank_line_after_namespace' => true,
-    'blank_line_after_opening_tag' => true,
-    'blank_line_before_statement' => [
-        'statements' => [
-            'break', 'case', 'continue', 'declare', 'default', 'do', 'for',
-            'if', 'foreach', 'return', 'switch', 'try', 'while',
-        ],
-    ],
-    'cast_spaces' => ['space' => 'single'],
-    'class_attributes_separation' => true,
-    'combine_consecutive_issets' => true,
-    'combine_consecutive_unsets' => true,
-    'compact_nullable_type_declaration' => true,
-    'concat_space' => ['spacing' => 'one'],
-    'fully_qualified_strict_types' => true,
-    'increment_style' => ['style' => 'pre'],
-    'linebreak_after_opening_tag' => true,
-    'list_syntax' => ['syntax' => 'short'],
-    'lowercase_cast' => true,
-    'lowercase_static_reference' => true,
-    'magic_constant_casing' => true,
-    'magic_method_casing' => true,
-    'multiline_comment_opening_closing' => true,
-    'multiline_whitespace_before_semicolons' => [
-        'strategy' => 'no_multi_line',
-    ],
-    'native_function_casing' => true,
-    'native_type_declaration_casing' => true,
-    'new_with_parentheses' => true,
-    'no_alternative_syntax' => true,
-    'no_blank_lines_after_class_opening' => true,
-    'no_blank_lines_after_phpdoc' => true,
-    'no_empty_statement' => true,
-    'no_extra_blank_lines' => true,
-    'no_leading_import_slash' => true,
-    'no_leading_namespace_whitespace' => true,
-    'no_mixed_echo_print' => ['use' => 'echo'],
-    'no_null_property_initialization' => true,
-    'no_short_bool_cast' => true,
-    'no_singleline_whitespace_before_semicolons' => true,
-    'no_spaces_around_offset' => true,
-    'no_superfluous_phpdoc_tags' => false,
-    'no_superfluous_elseif' => true,
-    'no_trailing_comma_in_singleline' => true,
-    'no_unneeded_control_parentheses' => true,
-    'no_unneeded_braces' => true,
-    'no_unset_cast' => true,
-    'no_unused_imports' => true,
-    'no_useless_else' => true,
-    'no_useless_return' => true,
-    'no_whitespace_before_comma_in_array' => true,
-    'no_whitespace_in_blank_line' => true,
-    'normalize_index_brace' => true,
-    'nullable_type_declaration_for_default_null_value' => true,
-    'object_operator_without_whitespace' => true,
-    'ordered_class_elements' => [
-        'order' => [
-            'use_trait', 'constant_public', 'constant_protected',
-            'constant_private', 'property_public_static', 'property_protected_static',
-            'property_private_static', 'property_public', 'property_protected',
-            'property_private', 'method_public_static', 'method_protected_static',
-            'method_private_static', 'construct', 'destruct', 'phpunit',
-            'method_public', 'method_protected', 'method_private', 'magic',
-        ],
-        'sort_algorithm' => 'none',
-    ],
-    'php_unit_fqcn_annotation' => true,
-    'php_unit_method_casing' => ['case' => 'camel_case'],
-    'phpdoc_add_missing_param_annotation' => ['only_untyped' => false],
-    'phpdoc_align' => ['align' => 'left'],
-    'phpdoc_line_span' => [
-        'const' => 'multi',
-        'method' => 'multi',
-        'property' => 'multi',
-    ],
-    'phpdoc_no_access' => true,
-    'phpdoc_no_empty_return' => true,
-    'phpdoc_no_useless_inheritdoc' => true,
-    'phpdoc_order' => true,
-    'phpdoc_scalar' => true,
-    'phpdoc_single_line_var_spacing' => true,
-    'phpdoc_to_comment' => false,
-    'phpdoc_trim' => true,
-    'phpdoc_trim_consecutive_blank_line_separation' => true,
-    'phpdoc_var_without_name' => true,
-    'protected_to_private' => true,
-    'return_assignment' => false,
-    'return_type_declaration' => ['space_before' => 'one'],
-    'semicolon_after_instruction' => true,
-    'short_scalar_cast' => true,
-    'simplified_null_return' => true,
-    'single_quote' => true,
-    'single_line_comment_style' => true,
-    'ternary_operator_spaces' => true,
-    'ternary_to_null_coalescing' => true,
-    'type_declaration_spaces' => true,
-    'trim_array_spaces' => true,
-    'unary_operator_spaces' => true,
-    'whitespace_after_comma_in_array' => true,
-])->setFinder($finder);
+return $config;

.phplint.yml

@@ -0,0 +1,8 @@
+path: ./
+jobs: 10
+cache-dir: runtime/.phplint.cache/
+extensions:
+    - php
+exclude:
+    - vendor/
+    - runtime/

composer.json

@@ -44,6 +44,7 @@
     "require-dev": {
         "friendsofphp/php-cs-fixer": "^3.73",
         "phpbench/phpbench": "^1.0",
+        "overtrue/phplint": "^9.6.2",
         "phpstan/extension-installer": "^1.0",
         "phpstan/phpstan": "^2.0",
         "phpstan/phpstan-phpunit": "^2.0",
@@ -89,6 +90,7 @@
             "@putenv PHP_CS_FIXER_IGNORE_ENV=1",
             "php-cs-fixer fix --config=.php-cs-fixer.dist.php"
         ],
+        "phplint": "phplint",
         "test": "phpunit"
     },
     "config": {

docs/neural-network/activation-functions/elu.md

@@ -1,23 +1,35 @@
-<span style="float:right;"><a href="https://github.com/RubixML/ML/blob/master/src/NeuralNet/ActivationFunctions/ELU.php">[source]</a></span>
+<span style="float:right;"><a href="https://github.com/RubixML/ML/blob/master/src/NeuralNet/ActivationFunctions/ELU/ELU.php">[source]</a></span>
 
 # ELU
 *Exponential Linear Units* are a type of rectifier that soften the transition from non-activated to activated using the exponential function. As such, ELU produces smoother gradients than the piecewise linear [ReLU](relu.md) function.
 
 $$
-{\displaystyle ELU = {\begin{cases}\alpha \left(e^{x}-1\right)&{\text{if }}x\leq 0\\x&{\text{if }}x>0\end{cases}}}
+\text{ELU}(x) =
+\begin{cases}
+\alpha \left(e^{x}-1\right) & \text{if } x \leq 0 \\
+x & \text{if } x > 0
+\end{cases}
 $$
 
 ## Parameters
 | # | Name | Default | Type | Description |
 |---|---|---|---|---|
 | 1 | alpha | 1.0 | float | The value at which leakage will begin to saturate. Ex. alpha = 1.0 means that the output will never be less than -1.0 when inactivated. |
 
+## Size and Performance
+ELU is a simple function and is well-suited for deployment on resource-constrained devices or when working with large neural networks.
+
+## Plots
+<img src="../../images/activation-functions/elu.png" alt="ELU Function" width="500" height="auto">
+
+<img src="../../images/activation-functions/elu-derivative.png" alt="ELU Derivative" width="500" height="auto">
+
 ## Example
 ```php
-use Rubix\ML\NeuralNet\ActivationFunctions\ELU;
+use Rubix\ML\NeuralNet\ActivationFunctions\ELU\ELU;
 
 $activationFunction = new ELU(2.5);
 ```
 
 ## References
-[^1]: D. A. Clevert et al. (2016). Fast and Accurate Deep Network Learning by Exponential Linear Units.
+[1]: D. A. Clevert et al. (2016). Fast and Accurate Deep Network Learning by Exponential Linear Units.

docs/neural-network/activation-functions/gelu.md

@@ -1,14 +1,26 @@
-<span style="float:right;"><a href="https://github.com/RubixML/ML/blob/master/src/NeuralNet/ActivationFunctions/GELU.php">[source]</a></span>
+<span style="float:right;"><a href="https://github.com/RubixML/ML/blob/master/src/NeuralNet/ActivationFunctions/GELU/GELU.php">[source]</a></span>
 
 # GELU
 Gaussian Error Linear Units (GELUs) are rectifiers that are gated by the magnitude of their input rather than the sign of their input as with ReLU variants. Their output can be interpreted as the expected value of a neuron with random dropout regularization applied.
 
+$$
+\text{GELU}(x) = 0.5 \cdot x \left(1 + \tanh\left(\sqrt{\frac{2}{\pi}} \left(x + 0.044715 \cdot x^3\right)\right)\right)
+$$
+
 ## Parameters
 This activation function does not have any parameters.
 
+## Size and Performance
+GELU is computationally more expensive than simpler activation functions like ReLU due to its use of hyperbolic tangent and exponential calculations. The implementation uses an approximation formula to improve performance, but it still requires more computational resources. Despite this cost, GELU has gained popularity in transformer architectures and other deep learning models due to its favorable properties for training deep networks.
+
+## Plots
+<img src="../../images/activation-functions/gelu.png" alt="GELU Function" width="500" height="auto">
+
+<img src="../../images/activation-functions/gelu-derivative.png" alt="GELU Derivative" width="500" height="auto">
+
 ## Example
 ```php
-use Rubix\ML\NeuralNet\ActivationFunctions\GELU;
+use Rubix\ML\NeuralNet\ActivationFunctions\GELU\GELU;
 
 $activationFunction = new GELU();
 ```

docs/neural-network/activation-functions/hard-sigmoid.md

@@ -0,0 +1,29 @@
+<span style="float:right;"><a href="https://github.com/RubixML/ML/blob/master/src/NeuralNet/ActivationFunctions/HardSigmoid/HardSigmoid.php">[source]</a></span>
+
+# Hard Sigmoid
+A piecewise linear approximation of the sigmoid function that is computationally more efficient. The Hard Sigmoid function has an output value between 0 and 1, making it useful for binary classification problems.
+
+$$
+\text{HardSigmoid}(x) = \max\left(0,\min\left(1, 0.2x + 0.5\right)\right)
+$$
+
+## Parameters
+This activation function does not have any parameters.
+
+## Size and Performance
+Hard Sigmoid has a minimal memory footprint compared to the standard Sigmoid function, as it uses simple arithmetic operations (multiplication, addition) and comparisons instead of expensive exponential calculations. This makes it particularly well-suited for mobile and embedded applications or when computational resources are limited.
+
+## Plots
+<img src="../../images/activation-functions/hard-sigmoid.png" alt="Hard Sigmoid Function" width="500" height="auto">
+
+<img src="../../images/activation-functions/hard-sigmoid-derivative.png" alt="Hard Sigmoid Derivative" width="500" height="auto">
+
+## Example
+```php
+use Rubix\ML\NeuralNet\ActivationFunctions\HardSigmoid\HardSigmoid;
+
+$activationFunction = new HardSigmoid();
+```
+
+## References
+[1]: https://en.wikipedia.org/wiki/Hard_sigmoid