On: Inside Firefox’s DNS-over-HTTPS engine

[textmode]

Here is a brief demonstration of how "DOH DNS servers" can be useful. Nevermind the idea of applications having their own DNS caches.

1. fetch page of html, e.g., hn front page

   curl https://news.ycombinator.com > 1.htm

2. extract urls from 1.htm

   yyt < 1.htm > 1.txt

(example scanner "yyt" provided below as t.l)

3. convert urls to hostnames

   g=1.txt k 1

(example script provided below as "1.k")

4. retrieve json dns data from doh dns server, efficiently, over a single connection

   see https://news.ycombinator.com/item?id=17228745

5. convert json dns data to csv

   see https://news.ycombinator.com/item?id=17228473

6. import csv into database e.g. sqlite3, kdb+, export to /etc/hosts, export to zonefile for localhost auth dns server, etc.

now, when user is reading hn front page, no dns lookups are needed. user already has the dns data. there is no network usage for dns requests, increasing hn front page browsing speed for the user. there are no piecemeal dns requests sent, increasing user privacy.

7. track ip address changes over time, compare answers from different caches, etc. retrieve type 2 (NS) instead of type 1 (A) records, then compare to NS records provided in public zonefiles from icann, public internet scans, etc.

cat t.l

    #define p printf("%s\n",yytext);
   %%
   \200|\201|\204|\223|\224|\230|\231|\234|\235
   http:\/\/[^ \n\r<>"#'|]* p;
   https:\/\/[^ \n\r<>"#'|]* p;
   ftp:\/\/[^ \n\r<>"#'|]* p;
   .|\n
   %%
   int main(){ yylex();}
   int yywrap(){}

    /* compile with something like: 
    flex -Crfa -8 -i t.l 
    cc  -pipe lex.yy.c -static -o yyt 
    */

cat 1.k

   /k3 (novice level)
   /usage: g=f k 1 where f is list of urls
   h0:_getenv "g";h1:0:h0; h1:{:[(#h1[x] _ss "://")>0;h1[x];_exit 1]}'!#h1;h1:{*((h1[x] _ss "://[^/]")+3) _ h1[x]}'!#h1;h2:{h1[x] _ss "[^a-z^A-Z^0-9^.^-]"};h3:{*h2[x]};h1:{h3[x]#h1[x]}'!#h1;h1:?:/h1;h0 0:h1;
   \\

[textmode → textmode]

correction to 1.k: delete empty line 1 from list of hostnames

   / novice alert. this is probably 3x larger than necessary
   h0:_getenv "g";h1:0:h0;h1:{:[(#h1[x] _ss "://")>0;h1[x];_exit 1]}'!#h1;h1:{*((h1[x] _ss "://[^/]")+3) _ h1[x]}'!#h1;h2:{h1[x] _ss "[^a-z^A-Z^0-9^.^-]"};h3:{*h2[x]};h1:{h3[x]#h1[x]}'!#h1;h1:?:/h1;if[0=#h1[0];h1:h1 _di 0];h0 0:h1;

\\

[textmode → textmode]

   ftp -4o 1.htm $1;
   fetch -4o 1.htm $1;
   wget -4o 1.htm $1;
   curl -4o 1.htm $1;
   
   exec k 1d \
   |exec tcs cloudflare-dns.com cloudflare-dns.com >1.json;
   yyf < 1.json >1.csv;
   #t=list exec k j;
   #t=hosts exec k j;
   t=db exec k j >1.csv;
   #t=zone exec k j;

   \\

   k).Q.fs[{`t insert +:`ts`ip`hn!("ZSS";",")0:x}]`:1.csv
   k)t

   \\

   / 1d.k
   / urls, hosts
   a:0:"1.htm";
   a:,/$a;
   a:_ssr[a;"\42";""];
   a:_ssr[a;"http://";"https://"];
   a:_ssr[a;"src=//";"https://"];
   b:a _ss "https://";
   a:b _ a;
   b:{a[x] _ss "[^a-z^A-Z^0-9^.^-]"}'!#a / fail: https://example.com-
   c:{1#(3_ b[x])}'!#b;
   b:{c[x]#a[x]}'!#c;
   b:?:/b;
   b:{8_ b[x] }'!#b;
   / http
   a:"GET /dns-query?ct=application/dns-json&name=";
   c:"&type=1 HTTP/1.1\r\nHost: cloudflare-dns.com\r\nConnection: ";
   d:"keep-alive";
   e:"close";
   f:"\r\n\r\n"; 
   g:(#b)-1;
   h:{a,b[x],c,d,f}'!g;
   i:a,b[g],c,e,f;

   `0:,/$h,i;

   \\

   int main(int argc, char **argv){
   char *b[17];
   b[0]="/usr/bin/openssl";
   b[1]="s_client";
   b[2]="-tls1_2";
   b[3]="-no_ssl2";
   b[4]="-no_ssl3";
   b[5]="-ign_eof";
   b[6]="-no_ticket";
   b[7]="-tlsextdebug";
   b[8]="-servername";
   b[9]=argv[2];
   b[10]="-verify";
   b[11]="9";
   /* -host, -port removed from manual */
   /* but still found in s_client.c */
   b[12]="-host"; 
   b[13]=argv[1];
   b[14]="-port";
   b[15]="443";
   b[16]=(void *)0;
   execve("/usr/bin/openssl",b,(void *)0);

   }



   /j.k

   j:("SSSSSS";",")0:"1.csv";
   k:_getenv "t";
   if[k _sm "list";`0:{,/$j[4;x]}'!#j[5]];
   if[k _sm "hosts";`0:{,/$j[5;x]," ",j[4;x]}'!#j[5]];
   if[k _sm "db";`0:{,/$j[2;x],".",j[1;x],".",j[0;x],"T",j[3;x],",",j[5;x],",",j[4;x]}'!#j[2]];
   if[k _sm "zone";`0:{,/$".",j[4;x],"\n&",j[4;x],".:127.0.0.1:5\n=",j[4;x],".:",j[5;x],":5"}'!#j[2]];
   \\

   \\

   /* f.l */

    #define echo ECHO
    #define jmp BEGIN
    #define p printf
    #define nl p("\n")
    #define s p(",")
   %s xa xb xc xd xx xy xz
   xa "\"Question\":[{\"name\": \""
   ya ", \"type\": 1}],"
   x1 "\"type\": 1" 
   x0 "\"type\": "[^1]","
   yb "\"data\": \""
   xw "Date: "
   xx \"\},\{\"
   xy \"\}\]\}
   xz "Sun, "|"Mon, "|"Tues, "|"Wed, "|"Thu, "|"Fri, "|"Sat, "
   %%
   {xw} jmp xz;
   <xz>"GMT" jmp xy;
   <xz>{xz}
   <xz>" Jan " p(",01,");
   <xz>" Feb " p(",02,");
   <xz>" Mar " p(",03,");
   <xz>" Apr " p(",04,");
   <xz>" May " p(",05,");
   <xz>" Jun " p(",06,");
   <xz>" Jul " p(",07,");
   <xz>" Aug " p(",08,");
   <xz>" Sep " p(",09,");
   <xz>" Oct " p(",10,");
   <xz>" Nov " p(",11,");
   <xz>" Dec " p(",12,");
   <xz>\40 s;
   <xz>. echo;
   <xy>{xa} jmp xa; 
   <xa>{ya} s;jmp xb;
   <xa>\.\"
   <xa>. echo;
   <xb>{x0} jmp xx;
   <xx>{xy} nl;jmp 0;
   <xx>{xx} jmp xb;
   <xb>{x1} jmp xc; 
   <xb>. 
   <xc>{yb} jmp xd;
   <xc>.  
   <xd>\"\} nl;jmp 0;
   <xd>. echo;
   \n
   .
   %%
   int main(){ yylex();}
   int yywrap(){}

[textmode → textmode]

fix: accomodate nxdomain, servfail, etc.

    /* f.l */
    #define echo ECHO
    #define jmp BEGIN
    #define p printf
    #define nl p("\n")
    #define s p(",")
   %s xz xy xa xb xx xc xd  
   xw "Date: "
   xz "Sun, "|"Mon, "|"Tues, "|"Wed, "|"Thu, "|"Fri, "|"Sat, "
   xa "\"Question\":[{\"name\": \""
   ya ", \"type\": 1}],"
   za ", \"type\": 1}]}"
   x0 "\"type\": "[^1]","
   xy \"\}\]\}
   xx \"\},\{\"
   x1 "\"type\": 1" 
   yb "\"data\": \""
   %%
   {xw} jmp xz;
   <xz>"GMT" jmp xy;
   <xz>{xz}
   <xz>" Jan " p(",01,");
   <xz>" Feb " p(",02,");
   <xz>" Mar " p(",03,");
   <xz>" Apr " p(",04,");
   <xz>" May " p(",05,");
   <xz>" Jun " p(",06,");
   <xz>" Jul " p(",07,");
   <xz>" Aug " p(",08,");
   <xz>" Sep " p(",09,");
   <xz>" Oct " p(",10,");
   <xz>" Nov " p(",11,");
   <xz>" Dec " p(",12,");
   <xz>\40 s;
   <xz>. echo;
   <xy>{xa} jmp xa; 
   <xa>{ya} s;jmp xb;
   <xa>{za} s;nl;jmp 0;
   <xa>\.\"
   <xa>. echo;
   <xb>{x0} jmp xx;
   <xx>{xy} nl;jmp 0;
   <xx>{xx} jmp xb;
   <xb>{x1} jmp xc; 
   <xb>. 
   <xc>{yb} jmp xd;
   <xc>.  
   <xd>\"\} nl;jmp 0;
   <xd>. echo;
   \n
   .
   %%
   int main(){ yylex();}
   int yywrap(){}