feat(flow-php/web): playground examples read from shared datasets

web/landing/content/examples/topics/data_frame/data_reading/csv/code.php

@@ -9,7 +9,7 @@
 
 data_frame()
     ->read(from_csv(
-        __DIR__ . '/input/dataset.csv',
+        __DIR__ . '/data/orders.csv',
         with_header: true,
         empty_to_null: true,
         separator: ',',

web/landing/content/examples/topics/data_frame/data_reading/excel/code.php

@@ -9,7 +9,7 @@
 
 data_frame()
     ->read(from_excel(
-        __DIR__ . '/input/dataset.xlsx',
+        __DIR__ . '/data/orders.xlsx',
     ))
     ->collect()
     ->write(to_output(truncate: false))

web/landing/content/examples/topics/data_frame/data_reading/floe/_meta.yaml

@@ -0,0 +1,2 @@
+priority: 0
+hidden: false

web/landing/content/examples/topics/data_frame/data_reading/floe/code.php

@@ -0,0 +1,14 @@
+<?php
+
+declare(strict_types=1);
+
+use function Flow\ETL\DSL\{data_frame, to_output};
+use function Flow\Floe\DSL\from_floe;
+
+require __DIR__ . '/vendor/autoload.php';
+
+data_frame()
+    ->read(from_floe(__DIR__ . '/data/orders.floe'))
+    ->collect()
+    ->write(to_output(truncate: false))
+    ->run();

web/landing/content/examples/topics/data_frame/data_reading/floe/description.md

@@ -0,0 +1 @@
+Extract data from Floe files. Floe is Flow's native self-describing binary format that stores the schema inside the file and evolves it across appended sections.

web/landing/content/examples/topics/data_frame/data_reading/floe/documentation.md

@@ -0,0 +1 @@
+- [Floe File Format](/documentation/components/core/floe)

web/landing/content/examples/topics/data_frame/data_reading/parquet/code.php

@@ -9,7 +9,7 @@
 
 data_frame()
     ->read(from_parquet(
-        __DIR__ . '/input/dataset.parquet',
+        __DIR__ . '/data/orders.parquet',
     ))
     ->collect()
     ->write(to_output(truncate: false))

web/landing/content/examples/topics/data_frame/data_writing/floe/_meta.yaml

@@ -0,0 +1,2 @@
+priority: 0
+hidden: false

web/landing/content/examples/topics/data_frame/data_writing/floe/code.php

@@ -0,0 +1,17 @@
+<?php
+
+declare(strict_types=1);
+
+use function Flow\ETL\Adapter\CSV\from_csv;
+use function Flow\ETL\DSL\{data_frame, overwrite};
+use function Flow\Floe\DSL\to_floe;
+
+require __DIR__ . '/vendor/autoload.php';
+
+data_frame()
+    ->read(from_csv(__DIR__ . '/data/orders.csv'))
+    ->limit(10)
+    ->collect()
+    ->saveMode(overwrite())
+    ->write(to_floe(__DIR__ . '/output.floe'))
+    ->run();

web/landing/content/examples/topics/data_frame/data_writing/floe/description.md

@@ -0,0 +1 @@
+Write data to Floe files. Floe is Flow's native self-describing binary format — the schema travels inside the file, so it reads back through the DataFrame API with no external metadata.

web/landing/content/examples/topics/data_frame/data_writing/floe/documentation.md

@@ -0,0 +1 @@
+- [Floe File Format](/documentation/components/core/floe)

web/landing/content/examples/topics/data_frame/data_writing/overwrite/code.php

@@ -8,7 +8,8 @@
 require __DIR__ . '/vendor/autoload.php';
 
 data_frame()
-    ->read(from_csv(__DIR__ . '/input/file.csv'))
+    ->read(from_csv(__DIR__ . '/data/orders.csv'))
+    ->limit(20)
     ->saveMode(overwrite())
     ->write(to_csv(__DIR__ . '/output/file.csv'))
     ->collect()
@@ -18,7 +19,7 @@
 data_frame()
     ->read(from_csv(__DIR__ . '/output/file.csv'))
     ->saveMode(overwrite())
-    ->drop('name')
+    ->drop('notes')
     ->write(to_csv(__DIR__ . '/output/file.csv'))
     ->collect()
     ->write(to_output(truncate: false))

web/landing/content/examples/topics/partitioning/partition_pruning/code.php

@@ -8,7 +8,7 @@
 require __DIR__ . '/vendor/autoload.php';
 
 data_frame()
-    ->read(from_csv(__DIR__ . '/input/color=*/sku=*/*.csv'))
+    ->read(from_csv(__DIR__ . '/data/partitioned/color=*/sku=*/*.csv'))
     ->filterPartitions(ref('color')->notEquals(lit('green')))
     ->collect()
     ->write(to_output(truncate: false))

web/landing/content/examples/topics/partitioning/path_partitions/code.php

@@ -7,7 +7,7 @@
 require __DIR__ . '/vendor/autoload.php';
 
 data_frame()
-    ->read(from_path_partitions(__DIR__ . '/input/color=*/sku=*/*.csv'))
+    ->read(from_path_partitions(__DIR__ . '/data/partitioned/color=*/sku=*/*.csv'))
     ->withEntry('path', ref('path')->strReplace(__DIR__, '/__ABSOLUTE_PATH__'))
     ->collect()
     ->write(to_output(truncate: false))

web/landing/content/examples/topics/partitioning/reading/basic/code.php

@@ -8,6 +8,6 @@
 require __DIR__ . '/vendor/autoload.php';
 
 data_frame()
-    ->read(from_csv(__DIR__ . '/input/color=*/sku=*/*.csv'))
+    ->read(from_csv(__DIR__ . '/data/partitioned/color=*/sku=*/*.csv'))
     ->write(to_output(truncate: false))
     ->run();

web/landing/content/examples/topics/schema/inferring/code.php

@@ -14,7 +14,7 @@
 
 if ($fs->status(path(__DIR__ . '/output/schema.json')) === null) {
     $schema = data_frame()
-        ->read(from_csv(__DIR__ . '/input/dataset.csv'))
+        ->read(from_csv(__DIR__ . '/data/orders.csv'))
         ->limit(100) // Limiting the number of rows to read will speed up the process but might bring less accurate results
         ->autoCast()
         ->schema();
@@ -28,7 +28,7 @@
 
 // Reading schemaless data formats with predefined schema can significantly improve performance
 data_frame()
-    ->read(from_csv(__DIR__ . '/input/dataset.csv', schema: $schema))
+    ->read(from_csv(__DIR__ . '/data/orders.csv', schema: $schema))
     ->collect()
     ->write(to_output(truncate: false, output: Output::rows_and_schema))
     ->run();

web/landing/phpunit.xml

@@ -13,7 +13,7 @@
   <php>
     <env name="APP_ENV" value="test"/>
     <server name="PANTHER_WEB_SERVER_ROUTER" value="../tests/router.php"/>
-    <server name="PANTHER_CHROME_ARGUMENTS" value="--no-first-run --no-default-browser-check --disable-search-engine-choice-screen --disable-features=Translate,InterestFeedContentSuggestions"/>
+    <server name="PANTHER_CHROME_ARGUMENTS" value="--window-size=1920,1400 --no-first-run --no-default-browser-check --disable-search-engine-choice-screen --disable-features=Translate,InterestFeedContentSuggestions"/>
     <server name="PANTHER_NO_SANDBOX" value="1"/>
   </php>
   <testsuites>

web/landing/src/Flow/Website/StaticSourceProvider/ExamplesSourceProvider.php

@@ -38,10 +38,27 @@ public function all(): array
                             'example' => $example,
                             'option' => $option,
                         ]);
+
+                        foreach ($this->examples->dataFiles($topic, $example, $option) as $path) {
+                            $sources[] = new Source('example_option_data', [
+                                'topic' => $topic,
+                                'example' => $example,
+                                'option' => $option,
+                                'path' => $path,
+                            ]);
+                        }
                     }
                 } else {
                     $sources[] = new Source('example', ['topic' => $topic, 'example' => $example]);
                     $sources[] = new Source('example_playground', ['topic' => $topic, 'example' => $example]);
+
+                    foreach ($this->examples->dataFiles($topic, $example) as $path) {
+                        $sources[] = new Source('example_data', [
+                            'topic' => $topic,
+                            'example' => $example,
+                            'path' => $path,
+                        ]);
+                    }
                 }
             }
         }

web/landing/templates/playground/_playground.html.twig

@@ -21,7 +21,13 @@ Parameters:
     'data/orders.json': asset('wasm/data/orders.json'),
     'data/orders.csv': asset('wasm/data/orders.csv'),
     'data/orders.jsonl': asset('wasm/data/orders.jsonl'),
-    'data/orders.parquet': asset('wasm/data/orders.parquet')
+    'data/orders.parquet': asset('wasm/data/orders.parquet'),
+    'data/orders.xlsx': asset('wasm/data/orders.xlsx'),
+    'data/orders.floe': asset('wasm/data/orders.floe'),
+    'data/partitioned/color=blue/sku=A/products.csv': asset('wasm/data/partitioned/color=blue/sku=A/products.csv'),
+    'data/partitioned/color=green/sku=A/products.csv': asset('wasm/data/partitioned/color=green/sku=A/products.csv'),
+    'data/partitioned/color=red/sku=A/products.csv': asset('wasm/data/partitioned/color=red/sku=A/products.csv'),
+    'data/partitioned/color=red/sku=B/products.csv': asset('wasm/data/partitioned/color=red/sku=B/products.csv')
 } %}
 {% for virtualPath, relativePath in data_files|default({}) %}
     {% if option is defined and option is not null %}

web/landing/tests/Flow/Website/Tests/Fixtures/WithDataFiles/topics/data_topic/_meta.yaml

@@ -0,0 +1,2 @@
+priority: 1
+hidden: false

web/landing/tests/Flow/Website/Tests/Fixtures/WithDataFiles/topics/data_topic/option_example/_meta.yaml

@@ -0,0 +1,2 @@
+priority: 2
+hidden: false

web/landing/tests/Flow/Website/Tests/Fixtures/WithDataFiles/topics/data_topic/option_example/option_1/_meta.yaml

@@ -0,0 +1,2 @@
+priority: 1
+hidden: false

web/landing/tests/Flow/Website/Tests/Fixtures/WithDataFiles/topics/data_topic/option_example/option_1/code.php

@@ -0,0 +1,5 @@
+<?php
+
+declare(strict_types=1);
+
+print 'example code file';

web/landing/tests/Flow/Website/Tests/Fixtures/WithDataFiles/topics/data_topic/option_example/option_1/input/nested/data.csv

@@ -0,0 +1,2 @@
+id,name
+2,B

web/landing/tests/Flow/Website/Tests/Fixtures/WithDataFiles/topics/data_topic/plain_example/_meta.yaml

@@ -0,0 +1,2 @@
+priority: 1
+hidden: false

web/landing/tests/Flow/Website/Tests/Fixtures/WithDataFiles/topics/data_topic/plain_example/code.php

@@ -0,0 +1,5 @@
+<?php
+
+declare(strict_types=1);
+
+print 'example code file';

web/landing/tests/Flow/Website/Tests/Fixtures/WithDataFiles/topics/data_topic/plain_example/input/dataset.csv

@@ -0,0 +1,2 @@
+id,name
+1,A

web/landing/tests/Flow/Website/Tests/Integration/ExamplesDatasetsTest.php

@@ -0,0 +1,78 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Flow\Website\Tests\Integration;
+
+use PHPUnit\Framework\TestCase;
+use RecursiveDirectoryIterator;
+use RecursiveIteratorIterator;
+use SplFileInfo;
+
+use function file_get_contents;
+use function glob;
+use function preg_match_all;
+use function sprintf;
+use function str_replace;
+
+final class ExamplesDatasetsTest extends TestCase
+{
+    public function test_every_example_references_a_shipped_and_registered_dataset(): void
+    {
+        $landingDir = __DIR__ . '/../../../../..';
+        // references and map keys are both rooted at "data/", which lives under assets/wasm/
+        $wasmDir = $landingDir . '/assets/wasm';
+
+        $registered = [];
+        preg_match_all(
+            "#'(data/[^']+)': asset\('wasm/(data/[^']+)'\)#",
+            (string) file_get_contents($landingDir . '/templates/playground/_playground.html.twig'),
+            $registered,
+        );
+
+        $checked = 0;
+
+        foreach (new RecursiveIteratorIterator(new RecursiveDirectoryIterator(
+            $landingDir . '/content/examples',
+            RecursiveDirectoryIterator::SKIP_DOTS,
+        )) as $file) {
+            /** @var SplFileInfo $file */
+            if ($file->getFilename() !== 'code.php') {
+                continue;
+            }
+
+            $references = [];
+            preg_match_all(
+                "#__DIR__ \. '/(data/[^']+)'#",
+                (string) file_get_contents($file->getPathname()),
+                $references,
+            );
+
+            foreach ($references[1] as $reference) {
+                $matches = glob($wasmDir . '/' . $reference);
+
+                static::assertNotEmpty($matches, sprintf(
+                    '%s references "%s" which the playground does not ship',
+                    $file->getPathname(),
+                    $reference,
+                ));
+
+                foreach ($matches as $match) {
+                    static::assertContains(
+                        str_replace($wasmDir . '/', '', $match),
+                        $registered[1],
+                        sprintf(
+                            '%s references "%s" which is not registered in the wasm_resources map',
+                            $file->getPathname(),
+                            $reference,
+                        ),
+                    );
+                }
+
+                $checked++;
+            }
+        }
+
+        static::assertGreaterThan(0, $checked, 'No dataset references were found to check');
+    }
+}