Merge branch 'ibraimgm-warc'

gabriel-vasile · Oct 4, 2019 · 84e752a · 84e752a
2 parents 6be6bfc + 0078266
commit 84e752a
Show file tree

Hide file tree

Showing 5 changed files with 28 additions and 2 deletions.
diff --git a/internal/matchers/archive.go b/internal/matchers/archive.go
@@ -76,3 +76,8 @@ func Rar(in []byte) bool {
 	}
 	return len(in) > 8 && (bytes.Equal(in[6:8], []byte{0x01, 0x00}) || in[6] == 0x00)
 }
+
+// Warc matches a Web ARChive file
+func Warc(in []byte) bool {
+	return bytes.HasPrefix(in, []byte("WARC/"))
+}
diff --git a/mime_test.go b/mime_test.go
@@ -48,6 +48,7 @@ var files = map[string]*node{
 	"djvu.djvu":   djvu,
 	"mobi.mobi":   mobi,
 	"lit.lit":     lit,
+	"warc.warc":   warc,
 
 	// images
 	"png.png":   png,

diff --git a/supported_mimes.md b/supported_mimes.md
@@ -1,4 +1,4 @@
-## 118 Supported MIME types
+## 119 Supported MIME types
 This file is automatically generated when running tests. Do not edit manually.
 
 Extension | MIME type
@@ -101,6 +101,7 @@ Extension | MIME type
 **tsv** | text/tab-separated-values
 **vcf** | text/vcard
 **ics** | text/calendar
+**warc** | application/warc
 **gz** | application/gzip
 **class** | application/x-java-applet; charset=binary
 **swf** | application/x-shockwave-flash

diff --git a/testdata/warc.warc b/testdata/warc.warc
@@ -0,0 +1,18 @@
+WARC/1.1
+WARC-Type: warcinfo
+WARC-Date: 2006-09-19T17:20:14Z
+WARC-Record-ID: <urn:uuid:d7ae5c10-e6b3-4d27-967d-34780c58ba39>
+Content-Type: application/warc-fields
+Content-Length: 381
+
+software: Heritrix 1.12.0 http://crawler.archive.org
+hostname: crawling017.archive.org
+ip: 207.241.227.234
+isPartOf: testcrawl-20050708
+description: testcrawl with WARC output
+operator: IA\_Admin
+http-header-user-agent:
+ Mozilla/5.0 (compatible; heritrix/1.4.0 +http://crawler.archive.org)
+format: WARC file version 1.1
+conformsTo:
+ http://iipc.github.io/warc-specifications/specifications/warc-format/warc-1.1/
diff --git a/tree.go b/tree.go
@@ -36,7 +36,7 @@ var (
 	ogg       = newNode("application/ogg", "ogg", matchers.Ogg, oggAudio, oggVideo)
 	oggAudio  = newNode("audio/ogg", "oga", matchers.OggAudio)
 	oggVideo  = newNode("video/ogg", "ogv", matchers.OggVideo)
-	txt       = newNode("text/plain", "txt", matchers.Txt, html, svg, xml, php, js, lua, perl, python, json, ndJson, rtf, tcl, csv, tsv, vCard, iCalendar)
+	txt       = newNode("text/plain", "txt", matchers.Txt, html, svg, xml, php, js, lua, perl, python, json, ndJson, rtf, tcl, csv, tsv, vCard, iCalendar, warc)
 	xml       = newNode("text/xml; charset=utf-8", "xml", matchers.Xml, rss, atom, x3d, kml, xliff, collada, gml, gpx, tcx, amf, threemf)
 	json      = newNode("application/json", "json", matchers.Json, geoJson)
 	csv       = newNode("text/csv", "csv", matchers.Csv)
@@ -132,4 +132,5 @@ var (
 	lit       = newNode("application/x-ms-reader", "lit", matchers.Lit)
 	sqlite3   = newNode("application/x-sqlite3", "sqlite", matchers.Sqlite)
 	dwg       = newNode("image/vnd.dwg", "dwg", matchers.Dwg)
+	warc      = newNode("application/warc", "warc", matchers.Warc)
 )