Add in job orchestration base attributes, add in Apache Airflow, upda…

…te mkdocs nav when new tech is added

README.md

@@ -19,6 +19,17 @@ Compare different technologies. No BS and all sources linked.
 5. Run `mkdocs build` to generate site
 6. Open [site/index.html](site/index.html) to see result
 
+Step 1 and 2  => Base data of different technologies
+Step 3-6      => Generate UI
+
+### JSON Schema
+
+JSON Schema for tech-diff can be found [here](spec/data-tech-compare-spec.json). You can import this schema into your IDE for
+validation of your YAML files. Links below show how you can import the schema:
+
+- [Intellij](https://www.jetbrains.com/help/idea/json.html#ws_json_schema_add_custom)
+- [VSCode](https://code.visualstudio.com/docs/languages/json#_json-schemas-and-settings)
+
 ## How to contribute
 
 - Add new high level type of technology (i.e. databases, object stores, messaging system)
@@ -89,6 +100,7 @@ project itself (either from documentation or source code).
   - SodaCL
   - Pydantic
   - Deequ
+  - Great Expectations
 
 ## Cloud Providers
 

build.gradle

@@ -35,26 +35,41 @@ tasks.register("generate") {
         cfg.setDefaultEncoding("UTF-8")
         var template = cfg.getTemplate("template-index.ftl")
 
-        techTypes.forEach { tech ->
-            var baseDir = new File("tech/$tech")
-            var root = new HashMap()
-            var innerTechTypes = new ArrayList()
+        new File("$workingDir/tech").eachFileRecurse(groovy.io.FileType.DIRECTORIES) { dir ->
+            if (dir.name != "templates") {
+                var tech = dir.name
+                var baseDir = new File("tech/$tech")
+                var root = new HashMap()
+                var innerTechTypes = new ArrayList()
 
-            baseDir.traverse(type: FILES, sort: sortByName) { file ->
-                var parsedYaml = yaml.load(file.getText())
-                println("Processing: ${parsedYaml["name"]}")
-                root.put(parsedYaml["name"], parsedYaml)
-                innerTechTypes.add(parsedYaml["name"])
-            }
-            root.put("main_tech", tech)
-            root.put("tech_types", innerTechTypes)
-            var outFile = new File("$workingDir/docs/$tech/index.md")
-            if (!outFile.exists()) {
-                outFile.createNewFile()
+                baseDir.traverse(type: FILES, sort: sortByName) { file ->
+                    var parsedYaml = yaml.load(file.getText())
+                    println("Processing: ${parsedYaml["name"]}")
+                    root.put(parsedYaml["name"], parsedYaml)
+                    innerTechTypes.add(parsedYaml["name"])
+                }
+                root.put("high_level_tech_name", tech)
+                root.put("tech_types", innerTechTypes)
+                var outFile = new File("$workingDir/docs/$tech/index.md")
+                if (!outFile.parentFile.exists()) {
+                    outFile.parentFile.mkdirs()
+                }
+                if (!outFile.exists()) {
+                    outFile.createNewFile()
+                }
+                var out = new FileWriter(outFile)
+                template.process(root, out)
+                out.close()
+
+                var mkdocsFile = new File("$workingDir/mkdocs.yml")
+                if (!mkdocsFile.text.contains("'$tech/index.md'")) {
+                    var cleanTechName = tech.replace("-", " ").capitalize()
+                    var updatedMkdocs = mkdocsFile.text.concat("\n  - $cleanTechName: '$tech/index.md'")
+                    var mkdocsOut = new FileWriter(mkdocsFile)
+                    mkdocsOut.write(updatedMkdocs)
+                    mkdocsOut.close()
+                }
             }
-            var out = new FileWriter(outFile)
-            template.process(root, out)
-            out.close()
         }
     }
 }

docs/job-orchestration/index.md

@@ -0,0 +1,60 @@
+---
+hide:
+- navigation
+- toc
+---
+# Job orchestration
+
+[![Apache Airflow logo](../assets/logo/airflow.png){: style="height:30px;width:30px" .lg align-left } Apache Airflow](){ .md-button .toggle-vis data-column="1" }
+
+
+<table id="example" class="stripe hover row-border compact" style="width:100%">
+    <thead>
+        <tr>
+            <th>Attribute</th>
+            <th>Apache Airflow</th>
+        </tr>
+    </thead>
+    <tbody>
+        <tr>
+            <td><b>Name</b></td>
+            <td>Apache Airflow</td>
+        </tr>
+        <tr>
+            <td><b>Description</b></td>
+            <td>Apache Airflow is a platform to programmatically author, schedule, and monitor workflows.</td>
+        </tr>
+        <tr>
+            <td><b>License</b></td>
+            <td>Apache license 2.0</td>
+        </tr>
+        <tr>
+            <td><b>Source code</b></td>
+            <td><a href="https://github.com/apache/airflow">https://github.com/apache/airflow</a></td>
+        </tr>
+        <tr>
+            <td><b>Website</b></td>
+            <td><a href="https://airflow.apache.org/">https://airflow.apache.org/</a></td>
+        </tr>
+        <tr>
+            <td><b>Year created</b></td>
+            <td>2014</td>
+        </tr>
+        <tr>
+            <td><b>Company</b></td>
+            <td>Airbnb, Apache</td>
+        </tr>
+        <tr>
+            <td><b>Language support</b></td>
+            <td>python</td>
+        </tr>
+        <tr>
+            <td><b>Use cases</b></td>
+            <td>Workflow scheduling</td>
+        </tr>
+        <tr>
+            <td><b>Job orchestration</b></td>
+            <td>N/A</td>
+        </tr>
+    </tbody>
+</table>

mkdocs.yml

@@ -37,10 +37,6 @@ theme:
         icon: material/brightness-4
         name: Switch to system preference
 
-nav:
-  - Home: 'index.md'
-  - File: 'file/index.md'
-
 extra_javascript:
   - https://code.jquery.com/jquery-3.7.0.js
   - https://cdn.datatables.net/1.13.7/js/jquery.dataTables.min.js
@@ -81,3 +77,8 @@ markdown_extensions:
   - pymdownx.emoji:
       emoji_index: !!python/name:materialx.emoji.twemoji
       emoji_generator: !!python/name:materialx.emoji.to_svg
+
+nav:
+  - Home: 'index.md'
+  - File: 'file/index.md'
+  - Job orchestration: 'job-orchestration/index.md'

site/404.html

@@ -263,6 +263,23 @@
 
 
 
+
+
+
+
+    <li class="md-tabs__item">
+      <a href="/job-orchestration/" class="md-tabs__link">
+
+
+
+
+  Job orchestration
+
+      </a>
+    </li>
+
+
+
     </ul>
   </div>
 </nav>
@@ -360,6 +377,31 @@
 
 
 
+
+
+
+
+
+    <li class="md-nav__item">
+      <a href="/job-orchestration/" class="md-nav__link">
+
+
+  <span class="md-ellipsis">
+
+
+    Job orchestration
+
+
+
+  </span>
+
+
+
+      </a>
+    </li>
+
+
+
   </ul>
 </nav>
                   </div>

site/database/index.html

@@ -270,6 +270,23 @@
 
 
 
+
+
+
+
+    <li class="md-tabs__item">
+      <a href="../job-orchestration/" class="md-tabs__link">
+
+
+
+
+  Job orchestration
+
+      </a>
+    </li>
+
+
+
     </ul>
   </div>
 </nav>
@@ -367,6 +384,31 @@
 
 
 
+
+
+
+
+
+    <li class="md-nav__item">
+      <a href="../job-orchestration/" class="md-nav__link">
+
+
+  <span class="md-ellipsis">
+
+
+    Job orchestration
+
+
+
+  </span>
+
+
+
+      </a>
+    </li>
+
+
+
   </ul>
 </nav>
                   </div>

site/file/index.html

@@ -14,6 +14,8 @@
         <link rel="prev" href="..">
 
 
+        <link rel="next" href="../job-orchestration/">
+
 
 
 
@@ -274,6 +276,23 @@
 
 
 
+
+
+
+
+    <li class="md-tabs__item">
+      <a href="../job-orchestration/" class="md-tabs__link">
+
+
+
+
+  Job orchestration
+
+      </a>
+    </li>
+
+
+
     </ul>
   </div>
 </nav>
@@ -383,6 +402,31 @@
 
 
 
+
+
+
+
+
+    <li class="md-nav__item">
+      <a href="../job-orchestration/" class="md-nav__link">
+
+
+  <span class="md-ellipsis">
+
+
+    Job orchestration
+
+
+
+  </span>
+
+
+
+      </a>
+    </li>
+
+
+
   </ul>
 </nav>
                   </div>
@@ -927,6 +971,22 @@ <h1 id="file">File</h1>
           </a>
 
 
+
+          <a href="../job-orchestration/" class="md-footer__link md-footer__link--next" aria-label="Next: Job orchestration">
+            <div class="md-footer__title">
+              <span class="md-footer__direction">
+                Next
+              </span>
+              <div class="md-ellipsis">
+                Job orchestration
+              </div>
+            </div>
+            <div class="md-footer__button md-icon">
+
+              <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M4 11v2h12l-5.5 5.5 1.42 1.42L19.84 12l-7.92-7.92L10.5 5.5 16 11H4Z"/></svg>
+            </div>
+          </a>
+
       </nav>