Add in pairwise comparison for each technology

.cache/plugin/social/manifest.json

@@ -2,5 +2,59 @@
   "assets/images/social/index.png": "6f6b3add99756489361e006c0d2b9d5df173e51f",
   "assets/images/social/database/index.png": "8ee6be82cdcfdfff26456f23cf60be94de7debe8",
   "assets/images/social/file/index.png": "4e4a5927aa4ec4db27f94f53488933563c4e1480",
-  "assets/images/social/job_orchestration/index.png": "3116dff397b6c723639a3323379b83a3e547c6a8"
+  "assets/images/social/job_orchestration/index.png": "3116dff397b6c723639a3323379b83a3e547c6a8",
+  "assets/images/social/file/apache_avro/delta_lake.png": "427d66a5a4639458b3b831057f9a37065f556883",
+  "assets/images/social/file/apache_avro/apache_hudi.png": "cabba053648ba0bf4958f83601ed8ab7b66acf97",
+  "assets/images/social/file/apache_avro/apache_parquet.png": "d73b2c1870faf7f44cf9624ce5901a9a8d260a21",
+  "assets/images/social/file/apache_avro/apache_orc.png": "276930327a4bf690a7fde80eb46a871f1a968183",
+  "assets/images/social/file/apache_hudi/apache_avro.png": "0c9f1c669aa68e503c8688073d92afa3593cb432",
+  "assets/images/social/file/apache_avro/csv.png": "56a7b1a365c53fe7fb77a3dfe47a3d3c6fab2b7f",
+  "assets/images/social/file/apache_hudi/apache_iceberg.png": "9dedd53ac01f7cedd3ce57ba3ff9a4835b64dfde",
+  "assets/images/social/file/apache_hudi/csv.png": "f72543a1b8404e5428af61b0936fcf62452f426e",
+  "assets/images/social/file/apache_hudi/apache_parquet.png": "0d1be4ed23ee6ac765cb5f8daa66dcbf0ed39a43",
+  "assets/images/social/file/apache_hudi/delta_lake.png": "33a3430b5ce2791aafa7f86437af307633fe8f29",
+  "assets/images/social/file/apache_hudi/apache_orc.png": "a4820d434512952adc3c8cb18ada567fda918d20",
+  "assets/images/social/file/apache_iceberg/apache_avro.png": "3e03525abd3cca31a90941fbfc5b78cbcabc8614",
+  "assets/images/social/file/apache_iceberg/delta_lake.png": "1ccc6d4b6e31dc2cc75e28fe067172f9bc6b519f",
+  "assets/images/social/file/apache_iceberg/csv.png": "49a2f56b645fd74d2a8648afe94503332aa2827a",
+  "assets/images/social/file/apache_iceberg/apache_orc.png": "cd1fee6760ca971432b3e74c3196c5990c589dde",
+  "assets/images/social/file/apache_iceberg/apache_hudi.png": "a1fded2f19ebaea1c0e866687f70cdc60fbb370b",
+  "assets/images/social/file/apache_orc/apache_avro.png": "ac7be2402c16542962a38130d91f1b320f2079e1",
+  "assets/images/social/file/apache_iceberg/apache_parquet.png": "60288bd975fd7beb9cb89d6c21cff7330bb46efd",
+  "assets/images/social/file/apache_orc/apache_hudi.png": "e1d8ca7d66079db7a529345847522faee3e10dd4",
+  "assets/images/social/file/apache_orc/apache_parquet.png": "83b3e82df3d6b3c1336892cfadbf669430891568",
+  "assets/images/social/file/apache_orc/apache_iceberg.png": "1bd32650d514cb42dc8ed2a1477d6ab30ae1a5d4",
+  "assets/images/social/file/apache_orc/csv.png": "b5485c33a3cf8e0556158f0e0aa902584579f689",
+  "assets/images/social/file/apache_parquet/apache_avro.png": "0361ce4f3b7d3ebb909c0acd550660a679e7ceca",
+  "assets/images/social/file/apache_orc/delta_lake.png": "24d6a09f0ad9d508934a499ac2634bfaa6e48918",
+  "assets/images/social/file/apache_parquet/apache_hudi.png": "fc52722e3fe82486418a199478f50f892261dd0a",
+  "assets/images/social/file/apache_parquet/apache_orc.png": "dc25a4ce88e652fab0753c6207bf05744d060358",
+  "assets/images/social/file/apache_parquet/apache_iceberg.png": "fc83a3aa24ef3e6f08c58b2b321289cbc766d721",
+  "assets/images/social/file/apache_parquet/csv.png": "9c6c2048273227f7b35767dac79c2da86360d155",
+  "assets/images/social/file/csv/apache_avro.png": "9ff1d97ecfcad5d7a576a2c7a8365e8ad45ac342",
+  "assets/images/social/file/apache_parquet/delta_lake.png": "568a564dbe44352d7ec4f8b9a49968aed9ad1061",
+  "assets/images/social/file/csv/apache_hudi.png": "7ed128913330e3bf31a41a83ce91cb64a3d4ae64",
+  "assets/images/social/file/csv/apache_iceberg.png": "d30b4e98f78f28e9b14b64d95b171b0c46d306de",
+  "assets/images/social/file/csv/apache_orc.png": "b781e50566ee0a16c14d70005eb3fd9ab90c14a6",
+  "assets/images/social/file/csv/delta_lake.png": "c6e2fe1052f67f77c6967413137ec17324f2b9f8",
+  "assets/images/social/file/delta_lake/apache_avro.png": "bb918698cb55d24028921e1b32426c6d878699f4",
+  "assets/images/social/file/csv/apache_parquet.png": "af4eedacac5494f882d26a22aa4e3e1a22374a06",
+  "assets/images/social/file/delta_lake/apache_hudi.png": "e114a58bcb9fb80ae0e512dae72842cdf5f25edb",
+  "assets/images/social/file/delta_lake/apache_iceberg.png": "ddaf87de3152fd90f79c4d73be5bc09816c2fbb6",
+  "assets/images/social/file/delta_lake/csv.png": "fafff08c4c1d868abeaefa016cf5771986c1948c",
+  "assets/images/social/file/delta_lake/apache_orc.png": "a7468941d9f17420669bf872ec2d35620351f84c",
+  "assets/images/social/file/delta_lake/apache_parquet.png": "9f522dffbad9252092eefa75e1dac985ce084859",
+  "assets/images/social/job_orchestration/apache_airflow/mage.png": "bffce30fc7f0b8d60aff6f78805e879fe27b8407",
+  "assets/images/social/job_orchestration/apache_airflow/dagster.png": "44f87a006f73ca7a2a7a885a10f7d5bece6266f2",
+  "assets/images/social/job_orchestration/apache_airflow/prefect.png": "7014808abfd542cbf7d6a355b5f8fa45bbb964d8",
+  "assets/images/social/job_orchestration/dagster/mage.png": "7149d11fa3a7be835057644875b9857b1e21a2a8",
+  "assets/images/social/job_orchestration/dagster/prefect.png": "2f209e18725ce61922c4ec961c53d71c36a9f364",
+  "assets/images/social/job_orchestration/dagster/apache_airflow.png": "5e002c9e6741ff086547e2d8a236f208cb9e503a",
+  "assets/images/social/job_orchestration/mage/apache_airflow.png": "e09935472191bde1e0a9d6358f546665516860b1",
+  "assets/images/social/job_orchestration/mage/dagster.png": "305e81ba0739b4b5ad99c69e1daf416d3a62e260",
+  "assets/images/social/job_orchestration/mage/prefect.png": "86c42e2183b1422e2bdedc1fb90650aebbefc19c",
+  "assets/images/social/job_orchestration/prefect/apache_airflow.png": "f43fcc5bb0104e40e216aa86454e80579b66d7f7",
+  "assets/images/social/job_orchestration/prefect/mage.png": "04f7e8aec3dd2bcdf11490b3af5b645e9b12f43e",
+  "assets/images/social/job_orchestration/prefect/dagster.png": "1192837fffdecc9b08a4183db11ceae8fd9a0072",
+  "assets/images/social/file/apache_avro/apache_iceberg.png": "84cfbff4049b1f515de79f20c563614c070cdde0"
 }

build.gradle

@@ -26,21 +26,24 @@ tasks.register("generate") {
             a.name <=> b.name
         }
 
-        List<String> techTypes = List.of("file")
         LoaderOptions loaderOptions = new LoaderOptions()
         Yaml yaml = new Yaml(loaderOptions)
         var workingDir = "${rootProject.projectDir.absolutePath}"
         freemarker.template.Configuration cfg = new freemarker.template.Configuration(freemarker.template.Configuration.VERSION_2_3_32)
         cfg.setDirectoryForTemplateLoading(new File("$workingDir/tech/templates"))
         cfg.setDefaultEncoding("UTF-8")
-        var template = cfg.getTemplate("template-index.ftl")
+        var baseTechTemplate = cfg.getTemplate("template-index.ftl")
+        //update mkdocs file by getting the 'nav:' section and writing the updated site structure
+        var baseNavSection = "nav:\n  - Home: 'index.md'\n"
 
         new File("$workingDir/tech").eachFileRecurse(groovy.io.FileType.DIRECTORIES) { dir ->
             if (dir.name != "templates") {
                 var tech = dir.name
+                var cleanTechName = tech.replace("_", " ").capitalize()
                 var baseDir = new File("tech/$tech")
                 var root = new HashMap()
                 var innerTechTypes = new ArrayList()
+                baseNavSection += "  - $cleanTechName:\n    - '$tech/index.md'\n"
 
                 baseDir.traverse(type: FILES, sort: sortByName) { file ->
                     var parsedYaml = yaml.load(file.getText())
@@ -50,26 +53,56 @@ tasks.register("generate") {
                 }
                 root.put("high_level_tech_name", tech)
                 root.put("tech_types", innerTechTypes)
-                var outFile = new File("$workingDir/docs/$tech/index.md")
-                if (!outFile.parentFile.exists()) {
-                    outFile.parentFile.mkdirs()
+
+                // main file with all tech comparisons
+                var baseTechOutFile = new File("$workingDir/docs/$tech/index.md")
+                if (!baseTechOutFile.parentFile.exists()) {
+                    baseTechOutFile.parentFile.mkdirs()
                 }
-                if (!outFile.exists()) {
-                    outFile.createNewFile()
+                if (!baseTechOutFile.exists()) {
+                    baseTechOutFile.createNewFile()
                 }
-                var out = new FileWriter(outFile)
-                template.process(root, out)
+                var out = new FileWriter(baseTechOutFile)
+                baseTechTemplate.process(root, out)
                 out.close()
 
-                var mkdocsFile = new File("$workingDir/mkdocs.yml")
-                if (!mkdocsFile.text.contains("'$tech/index.md'")) {
-                    var cleanTechName = tech.replace("_", " ").capitalize()
-                    var updatedMkdocs = mkdocsFile.text.concat("\n  - $cleanTechName: '$tech/index.md'")
-                    var mkdocsOut = new FileWriter(mkdocsFile)
-                    mkdocsOut.write(updatedMkdocs)
-                    mkdocsOut.close()
-                }
+                // sub files with comparison with each pair
+                innerTechTypes.forEach(techType -> {
+                    baseNavSection += "    - $techType:\n"
+                    innerTechTypes.forEach(techType2 -> {
+                        var techTypeName = techType.toLowerCase().replaceAll(" ", "_")
+                        var techType2Name = techType2.toLowerCase().replaceAll(" ", "_")
+                        if (techTypeName != techType2Name) {
+                            baseNavSection += "      - $techType2: '$tech/$techTypeName/${techType2Name}.md'\n"
+
+                            var compareRoot = new HashMap()
+                            compareRoot.put("compare_two_tech", true)
+                            compareRoot.put("high_level_tech_name", tech)
+                            compareRoot.put("tech_types", [techType, techType2])
+                            compareRoot.put(techType, root[techType])
+                            compareRoot.put(techType2, root[techType2])
+
+                            var compareTwoOutFile = new File("$workingDir/docs/$tech/$techTypeName/${techType2Name}.md")
+                            if (!compareTwoOutFile.parentFile.exists()) {
+                                compareTwoOutFile.parentFile.mkdirs()
+                            }
+                            if (!compareTwoOutFile.exists()) {
+                                compareTwoOutFile.createNewFile()
+                            }
+                            var compareOut = new FileWriter(compareTwoOutFile)
+                            baseTechTemplate.process(compareRoot, compareOut)
+                            compareOut.close()
+                        }
+                    })
+                })
             }
         }
+
+        //update mkdocs file nav section with baseNavSection that will have updated site structure
+        var mkdocsFile = new File("$workingDir/mkdocs.yml")
+        var updatedMkdocs = mkdocsFile.text.replaceAll(/(?ms)nav:.*/, baseNavSection)
+        var mkdocsOut = new FileWriter(mkdocsFile)
+        mkdocsOut.write(updatedMkdocs)
+        mkdocsOut.close()
     }
 }