main source code

1 /// Author: Aziz Köksal
2 /// License: GPL3
3 /// $(Maturity average)
4 module main;
5 
6 import dil.parser.Parser;
7 import dil.lexer.Lexer,
8        dil.lexer.Funcs,
9        dil.lexer.Token,
10        dil.lexer.TokenSerializer;
11 import dil.ast.Declarations,
12        dil.ast.Expressions,
13        dil.ast.Node,
14        dil.ast.Visitor,
15        dil.ast.ASTPrinter,
16        dil.ast.ASTSerializer;
17 import dil.semantic.Module,
18        dil.semantic.Symbols,
19        dil.semantic.Pass1,
20        dil.semantic.Pass2,
21        dil.semantic.Passes;
22 import dil.code.Interpreter;
23 import dil.translator.PyTreeEmitter;
24 import dil.i18n.Messages;
25 import dil.String,
26        dil.Version,
27        dil.Diagnostics,
28        dil.SourceText,
29        dil.Compilation;
30 
31 import util.Path,
32        util.OptParser;
33 
34 import cmd.Command,
35        cmd.Compile,
36        cmd.Highlight,
37        cmd.Statistics,
38        cmd.ImportGraph,
39        cmd.DDoc;
40 
41 import Settings,
42        SettingsLoader,
43        common;
44 
45 import std.file,
46        std.conv,
47        std.datetime,
48        std.regex;
49 
50 version(TangoTrace) debug
51 import tango.core.tools.TraceExceptions;
52 
53 alias toInt = to!int;
54 
55 /// Entry function of DIL.
56 void main(cstring[] args)
57 {
58   debug if (args.length >= 2 && args[1] == "unittests")
59     return runUnittests();
60 
61   if (args.length == 0)
62     throw new Exception("main() received 0 arguments");
63 
64   // 1. Create a global compilation context.
65   auto globalCC = newCompilationContext();
66   auto diag = globalCC.diag;
67   // 2. Load the configuration file of DIL.
68   auto config = ConfigLoader(globalCC, diag, args[0]);
69   config.load();
70   diag.bundle = config.resourceBundle;
71   updateWithSettings(globalCC);
72 
73   if (diag.hasInfo())
74     return printErrors(diag);
75   if (args.length <= 1)
76     return printHelp("main", diag);
77 
78   // 3. Execute a command.
79   auto op = new OptParser(args[2..$]);
80   op.missingArgMessage = diag.bundle.msg(MID.MissingOptionArgument);
81   op.usageErrorMessage = diag.bundle.msg(MID.UsageError);
82   cstring command = args[1];
83 
84   switch (command)
85   {
86   case "c2",
87        "c", "compile":
88     if (!op.hasArgs)
89       return printHelp(command, diag);
90     bool useCommand2 = command == "c2";
91 
92     auto cmd = new CompileCommand();
93     auto cc = cmd.context = globalCC;
94     cmd.diag = diag;
95     cstring value;
96 
97     op.add({ return parseDebugOrVersion(op, cc); });
98     op.add("-I", value, { cc.importPaths ~= value; });
99     op.add("-J", value, { cc.includePaths ~= value; });
100     op.add("-m32", cmd.m32);
101     op.add("-m64", cmd.m64);
102     op.add("-of", value, { cmd.binOutput = value; });
103     op.add("-release", cc.releaseBuild);
104     op.add("-unittest", cc.unittestBuild, {
105       version(D2)
106         cc.addVersionId("unittest");
107     });
108     op.add("-d", cc.acceptDeprecated);
109     op.add("-ps", cmd.printSymbolTree);
110     op.add("-pm", cmd.printModuleTree);
111     op.add("-v", cmd.verbose);
112     if (!op.parseArgs())
113       return op.printUsageError();
114 
115     cmd.filePaths = op.argv; // Remaining arguments are file paths.
116 
117     Command cmd_ = cmd;
118     if (useCommand2)
119     { // Temporary code to test CompileCommand2.
120       auto cmd2 = new CompileCommand2();
121       cmd2.cc = cc;
122       cmd2.filePaths = cmd.filePaths;
123       cmd2.mm = cmd.moduleMan;
124       cmd2.verbose = cmd.verbose;
125       cmd_ = cmd2;
126     }
127 
128     // TODO:
129     //if (cmd.m32)
130     //{
131     //  cc.removeVersionId("D_InlineAsm_X86_64");
132     //  cc.removeVersionId("X86_64");
133     //  cc.addVersionId("D_InlineAsm_X86");
134     //  cc.addVersionId("X86");
135     //}
136     //if (cmd.m64)
137     //{
138     //  cc.addVersionId("D_InlineAsm_X86_64");
139     //  cc.addVersionId("X86_64");
140     //  cc.removeVersionId("D_InlineAsm_X86");
141     //  cc.removeVersionId("X86");
142     //}
143 
144     cmd_.run();
145 
146     diag.hasInfo() && printErrors(diag);
147     break;
148   case "print":
149     if (!op.hasArgs)
150       return printHelp(command, diag);
151     cstring path;
152     op.addDefault({ path = op.popArg(); });
153     op.parseArgs();
154     auto m = new Module(path, globalCC);
155     m.parse();
156     if (!m.hasErrors())
157     {
158       auto ap = new ASTPrinter(true, globalCC);
159       auto text = ap.print(m.root);
160       Printfln("{}", text);
161     }
162     diag.hasInfo() && printErrors(diag);
163     break;
164   case "pytree", "py":
165     if (op.argv.length < 2)
166       return printHelp(command, diag);
167 
168     class PyTreeCommand : Command
169     {
170       CompilationContext cc;
171       Path dest;
172       cstring format = "d_{0}.py";
173       cstring[] filePaths;
174 
175       override void run()
176       {
177         foreach (path; filePaths)
178         {
179           auto modul = new Module(path, cc);
180           lzy(log("parse: {}", path));
181           modul.parse();
182           if (modul.hasErrors())
183             continue;
184           auto py = new PyTreeEmitter(modul);
185           auto modFQN = modul.getFQN().replace('.', '_');
186           auto pckgName = modul.packageName.replace('.', '_');
187           auto modName = modul.moduleName;
188           auto fileName = Format(format, modFQN, pckgName, modName);
189           auto destPath = (dest/fileName)[];
190           lzy(log("emit:  {}", destPath));
191           destPath.write(py.emit());
192         }
193       }
194     }
195 
196     auto cmd = new PyTreeCommand();
197     cmd.dest = Path(op.popArg());
198     cmd.cc = globalCC;
199 
200     op.add("--fmt", cmd.format);
201     op.add("-v", cmd.verbose);
202     if (!op.parseArgs())
203       return op.printUsageError();
204     cmd.filePaths = op.argv; // Remaining arguments are file paths.
205 
206     cmd.run();
207 
208     diag.hasInfo() && printErrors(diag);
209     break;
210   case "ddoc", "d":
211     if (op.argv.length < 2)
212       return printHelp(command, diag);
213 
214     auto cmd = new DDocCommand();
215     cmd.destDirPath = op.popArg();
216     cmd.context = globalCC;
217     cmd.diag = diag;
218     cstring value;
219 
220     // Parse arguments.
221     op.add({ return parseDebugOrVersion(op, cmd.context); });
222     op.add("--xml", cmd.writeXML);
223     op.add("--raw", cmd.rawOutput);
224     op.add("-hl", cmd.writeHLFiles);
225     op.add("-i", cmd.includeUndocumented);
226     op.add("--inc-private", cmd.includePrivate);
227     op.add("-v", cmd.verbose);
228     op.add("--kandil", cmd.useKandil);
229     op.add("--report", cmd.writeReport);
230     op.add("-rx", value, { cmd.regexps ~= regex(value); });
231     op.add("-m", cmd.modsTxtPath);
232     if (!op.parseArgs())
233       return op.printUsageError();
234     foreach (arg; op.argv)
235       if (arg.length > 6 && String(arg[$-5..$]).ieql(".ddoc"))
236         cmd.macroPaths ~= arg;
237       else
238         cmd.filePaths ~= arg;
239 
240     cmd.run();
241     diag.hasInfo() && printErrors(diag);
242     break;
243   case "hl", "highlight":
244     if (!op.hasArgs)
245       return printHelp(command, diag);
246 
247     auto cmd = new HighlightCommand();
248     cmd.cc = globalCC;
249     cmd.diag = diag;
250 
251     bool dummy;
252     alias HO = HighlightCommand.Option;
253 
254     op.add("--syntax", dummy, { cmd.add(HO.Syntax); });
255     op.add("--xml", dummy, { cmd.add(HO.XML); });
256     op.add("--html", dummy, { cmd.add(HO.HTML); });
257     op.add("--lines", dummy, { cmd.add(HO.PrintLines); });
258     if (!op.parseArgs())
259       return op.printUsageError();
260     foreach (arg; op.argv)
261       if (!cmd.filePathSrc)
262         cmd.filePathSrc = arg;
263       else
264         cmd.filePathDest = arg;
265 
266     cmd.run();
267     diag.hasInfo() && printErrors(diag);
268     break;
269   case "importgraph", "igraph":
270     if (!op.hasArgs)
271       return printHelp(command, diag);
272 
273     auto cmd = new IGraphCommand();
274     cmd.context = globalCC;
275 
276     cstring value;
277     bool dummy;
278     alias IO = IGraphCommand.Option;
279 
280     op.add({ return parseDebugOrVersion(op, cmd.context); });
281     op.add("-I", value, { cmd.context.importPaths ~= value; });
282     op.add("-x", value, { cmd.regexps ~= value; });
283     op.add("-l", value, { cmd.levels = toInt(value); });
284     op.add("-si", value, { cmd.siStyle = value; });
285     op.add("-pi", value, { cmd.piStyle = value; });
286     op.add("--dot", dummy,   { cmd.add(IO.PrintDot); });
287     op.add("--paths", dummy, { cmd.add(IO.PrintPaths); });
288     op.add("--list", dummy,  { cmd.add(IO.PrintList); });
289     op.add("--hle", dummy,   { cmd.add(IO.HighlightCyclicEdges); });
290     op.add("--hlv", dummy,   { cmd.add(IO.HighlightCyclicVertices); });
291     op.add("--gbp", dummy,   { cmd.add(IO.GroupByPackageNames); });
292     op.add("--gbf", dummy,   { cmd.add(IO.GroupByFullPackageName); });
293     op.add("-i", dummy,      { cmd.add(IO.IncludeUnlocatableModules); });
294     op.add("-m", dummy,      { cmd.add(IO.MarkCyclicModules); });
295     op.addDefault({ cmd.filePath = op.popArg(); });
296     if (!op.parseArgs())
297       return op.printUsageError();
298 
299     cmd.run();
300     diag.hasInfo() && printErrors(diag);
301     break;
302   case "stats", "statistics":
303     if (!op.hasArgs)
304       return printHelp(command, diag);
305 
306     auto cmd = new StatsCommand();
307     cmd.cc = globalCC;
308 
309     op.add("--toktable", cmd.printTokensTable);
310     op.add("--asttable", cmd.printNodesTable);
311     if (!op.parseArgs())
312       return op.printUsageError();
313     cmd.filePaths = op.argv;
314 
315     cmd.run();
316     break;
317   case "tok", "tokenize":
318     if (!op.hasArgs)
319       return printHelp(command, diag);
320 
321     class TokenizeCommand : Command
322     {
323       CompilationContext cc;
324       cstring srcFilePath;
325       cstring separator = "\n";
326       bool ignoreWSToks, printWS, fromStdin;
327 
328       override void run()
329       {
330         auto sourceText = fromStdin ?
331           new SourceText("stdin", readStdin()) :
332           new SourceText(srcFilePath, true);
333 
334         auto lx = new Lexer(sourceText, cc.tables.lxtables, cc.diag);
335         lx.scanAll();
336 
337         if (cc.diag.hasInfo())
338           return;
339 
340         foreach (token; lx.tokenList[0..$-1])
341         {
342           if (token.kind == TOK.Newline || ignoreWSToks && token.isWhitespace)
343             continue;
344           if (printWS && token.ws)
345             Stdout(token.wsChars);
346           Stdout(token.text)(separator);
347         }
348       }
349     }
350 
351     auto cmd = new TokenizeCommand();
352     cmd.cc = globalCC;
353 
354     op.add("-s", cmd.separator);
355     op.add("-", cmd.fromStdin);
356     op.add("-i", cmd.ignoreWSToks);
357     op.add("-ws", cmd.printWS);
358     op.addDefault({ cmd.srcFilePath = op.popArg(); });
359     if (!op.parseArgs())
360       return op.printUsageError();
361 
362     cmd.run();
363 
364     diag.hasInfo() && printErrors(diag);
365     break;
366   case "dlexed", "dlx":
367     if (!op.hasArgs)
368       return printHelp(command, diag);
369 
370     class SerializeCommand : Command
371     {
372       CompilationContext cc;
373       cstring srcFilePath, outFilePath;
374       bool fromStdin;
375 
376       override void run()
377       {
378         auto sourceText = fromStdin ?
379           new SourceText("stdin", readStdin()) :
380           new SourceText(srcFilePath, true);
381 
382         auto lx = new Lexer(sourceText, cc.tables.lxtables, cc.diag);
383         lx.scanAll();
384 
385         if (cc.diag.hasInfo())
386           return;
387 
388         auto data = TokenSerializer.serialize(lx.tokenList);
389         if (outFilePath.length)
390           outFilePath.write(data);
391         else
392           Stdout(cast(cstring)data);
393       }
394     }
395 
396     auto cmd = new SerializeCommand();
397     cmd.cc = globalCC;
398 
399     op.add("-", cmd.fromStdin);
400     op.add("-o", cmd.outFilePath);
401     op.addDefault({ cmd.srcFilePath = op.popArg(); });
402     if (!op.parseArgs())
403       return op.printUsageError();
404 
405     cmd.run();
406 
407     diag.hasInfo() && printErrors(diag);
408     break;
409   case "serialize", "sz":
410     if (!op.hasArgs)
411       return printHelp(command, diag);
412 
413     class SerializeASTCommand : Command
414     {
415       CompilationContext cc;
416       cstring srcFilePath, outFilePath;
417       bool fromStdin;
418 
419       override void run()
420       {
421         auto sourceText = fromStdin ?
422           new SourceText("stdin", readStdin()) :
423           new SourceText(srcFilePath, true);
424 
425         auto mod = new Module(sourceText, cc);
426         mod.parse();
427 
428         if (cc.diag.hasInfo())
429           return;
430 
431         auto astWriter = new ASTSerializer();
432         const data = astWriter.serialize(mod);
433 
434         if (outFilePath.length)
435           outFilePath.write(data);
436         else
437           Stdout(cast(cstring)data);
438       }
439     }
440 
441     auto cmd = new SerializeASTCommand();
442     cmd.cc = globalCC;
443 
444     op.add("-", cmd.fromStdin);
445     op.add("-o", cmd.outFilePath);
446     op.addDefault({ cmd.srcFilePath = op.popArg(); });
447     if (!op.parseArgs())
448       return op.printUsageError();
449 
450     cmd.run();
451 
452     diag.hasInfo() && printErrors(diag);
453     break;
454   case "profile":
455     if (!op.hasArgs)
456       break;
457     const(String)[] filePaths = (args[2] == "dstress") ?
458       String(cast(cstring)"dstress_files".read()).split('\0') :
459       toStrings(args[2..$]);
460 
461     auto tables = globalCC.tables.lxtables;
462 
463     static void printSecs(TickDuration swatch) {
464       Printfln("Scanned in {}ms.", swatch.msecs);
465     }
466     auto mt = measureTime!(printSecs);
467 
468     foreach (filePath; filePaths)
469       new Lexer(new SourceText(filePath[], true), tables).scanAll();
470     break;
471   case "settings", "set":
472     alias GS = GlobalSettings;
473     cstring
474       versionIds = String(";").join(GS.versionIds)[],
475       importPaths = String(";").join(GS.importPaths)[],
476       ddocPaths = String(";").join(GS.ddocFilePaths)[];
477 
478     cstring[string] settings = [
479       "DIL":config.executablePath,
480       "DILCONF":config.dilconfPath,
481       "DATADIR":GS.dataDir, "VERSION_IDS":versionIds,
482       "KANDILDIR":GS.kandilDir,
483       "IMPORT_PATHS":importPaths, "DDOC_FILES":ddocPaths,
484       "LANG_FILE":GS.langFile, "XML_MAP":GS.xmlMapFile,
485       "HTML_MAP":GS.htmlMapFile, "LEXER_ERROR":GS.lexerErrorFormat,
486       "PARSER_ERROR":GS.parserErrorFormat,
487       "SEMANTIC_ERROR":GS.semanticErrorFormat,
488       "TAB_WIDTH":Format("{}", GS.tabWidth)
489     ];
490 
491     cstring[] retrieve_settings;
492     if (args.length > 2)
493       retrieve_settings = args[2..$];
494 
495     if (retrieve_settings.length) // Print select settings.
496       foreach (name; retrieve_settings)
497       {
498         if (auto psetting = (name = String(name).toupper()[]) in settings)
499           Printfln("{}={}", name, *psetting);
500       }
501     else // Print all settings.
502       foreach (name; settings.keys.sort)
503         Printfln("{}={}", name, settings[name]);
504     break;
505   case "?", "h", "help":
506     printHelp(op.hasArgs ? op.popArg() : "", diag);
507     break;
508   case "-v", "v", "--version", "version":
509     Stdout(VERSION).newline;
510     break;
511   default:
512     printHelp("main", diag);
513   }
514 }
515 
516 /// Reads the standard input and returns its contents.
517 char[] readStdin()
518 {
519   import std.stdio;
520   char[] text;
521   foreach (buffer; stdin.byChunk(4096))
522     text ~= buffer;
523   return text;
524 }
525 
526 /// Available commands.
527 enum string COMMANDS =
528   "  help (?)
529   compile (c)
530   ddoc (d)
531   dlexed (dlx)
532   highlight (hl)
533   importgraph (igraph)
534   pytree (py)
535   serialize (sz)
536   settings (set)
537   statistics (stats)
538   tokenize (tok)
539   version (v)";
540 
541 /// Creates the global compilation context.
542 CompilationContext newCompilationContext()
543 {
544   /// Generates code: Adds a version id to a list if defined.
545   /// E.g.: version(X86) ids ~= "X86";
546   char[] defineVersionIds(string[] ids...)
547   {
548     char[] code;
549     foreach (id; ids)
550       code ~= "version(" ~ id ~ ") ids ~= \"" ~ id ~ "\";";
551     return code;
552   }
553 
554   auto cc = new CompilationContext;
555   string[] ids = ["DIL", "all"];
556   mixin(defineVersionIds(
557     "Windows", "Win32", "Win64", "linux", "OSX", "FreeBSD", "OpenBSD", "BSD",
558     "Solaris", "Posix", "AIX", "SkyOS", "SysV3", "SysV4", "Hurd", "Android",
559     "Cygwin", "MinGW", "X86", "X86_64", "ARM", "PPC", "PPC64", "IA64", "MIPS",
560     "MIPS64", "SPARC", "SPARC64", "S390", "S390X", "HPPA", "HPPA64", "SH",
561     "SH64", "Alpha", "LittleEndian", "BigEndian", "D_InlineAsm_X86",
562     "D_InlineAsm_X86_64", "D_LP64", "D_PIC", "D_SIMD"
563   ));
564 version(D2)
565   ids ~= "D_Version2";
566   foreach (id; ids)
567     cc.addVersionId(id);
568   return cc;
569 }
570 
571 /// Updates cc with variables from GlobalSettings.
572 void updateWithSettings(CompilationContext cc)
573 {
574   cc.importPaths = GlobalSettings.importPaths;
575   foreach (versionId; GlobalSettings.versionIds)
576     if (cc.tables.idents.isValidUnreservedIdentifier(versionId))
577       cc.addVersionId(versionId);
578 }
579 
580 /// Parses a debug or version command line option.
581 bool parseDebugOrVersion(ref OptParser op, CompilationContext context)
582 {
583   cstring val;
584   if (op.argv[0] == "-debug")
585     context.debugLevel = 1;
586   else if (op.parse("-debug", val))
587   {
588     if (val.length && isdigit(val[0]))
589       context.debugLevel = toInt(val);
590     else if (context.tables.idents.isValidUnreservedIdentifier(val))
591       context.addDebugId(val);
592   }
593   else if (op.parse("-version", val))
594   {
595     if (val.length && isdigit(val[0]))
596       context.versionLevel = toInt(val);
597     else if (context.tables.idents.isValidUnreservedIdentifier(val))
598       context.addVersionId(val);
599   }
600   else
601     return false;
602   return true;
603 }
604 
605 /// Prints the errors collected in diag.
606 void printErrors(Diagnostics diag)
607 {
608   foreach (info; diag.info)
609   {
610     cstring errorFormat;
611     auto tid = typeid(info);
612     if (tid is typeid(LexerError))
613       errorFormat = GlobalSettings.lexerErrorFormat;
614     else if (tid is typeid(ParserError))
615       errorFormat = GlobalSettings.parserErrorFormat;
616     else if (tid is typeid(SemanticError))
617       errorFormat = GlobalSettings.semanticErrorFormat;
618     else if (tid is typeid(Warning))
619       errorFormat = "{0}: Warning: {3}";
620     else if (tid is typeid(GeneralError))
621       errorFormat = "Error: {3}";
622     else
623       continue;
624     auto err = cast(Problem)info;
625     Stderr.formatln(errorFormat, err.filePath, err.loc, err.col, err.getMsg);
626   }
627 }
628 
629 /// Prints the help message of a command.
630 /// If the command wasn't found, the main help message is printed.
631 void printHelp(cstring command, Diagnostics diag)
632 {
633   cstring msg;
634   MID mid;
635   switch (command)
636   {
637   case "c", "compile":          mid = MID.HelpCompile;     goto Lcommon;
638   case "py", "pytree":          mid = MID.HelpPytree;      goto Lcommon;
639   case "ddoc", "d":             mid = MID.HelpDdoc;        goto Lcommon;
640   case "hl", "highlight":       mid = MID.HelpHighlight;   goto Lcommon;
641   case "importgraph", "igraph": mid = MID.HelpImportGraph; goto Lcommon;
642   case "tok", "tokenize":       mid = MID.HelpTokenize;    goto Lcommon;
643   case "dlexed", "dlx":         mid = MID.HelpDlexed;      goto Lcommon;
644   case "serialize", "sz":       mid = MID.HelpSerialize;   goto Lcommon;
645   case "stats", "statistics":   mid = MID.HelpStatistics;  goto Lcommon;
646   case "settings", "set":       mid = MID.HelpSettings;    goto Lcommon;
647   case "?", "h", "help":        mid = MID.HelpHelp;        goto Lcommon;
648   Lcommon:
649     msg = diag.msg(mid);
650     break;
651   case "main", "":
652     const COMPILED_WITH = __VENDOR__;
653     const COMPILED_VERSION = (V => V[0] ~ "." ~ V[1..4])(itoa(__VERSION__));
654     const COMPILED_DATE = __TIMESTAMP__;
655     msg = diag.formatMsg(MID.HelpMain, VERSION, COMMANDS, COMPILED_WITH,
656       COMPILED_VERSION, COMPILED_DATE);
657     break;
658   default:
659     msg = diag.formatMsg(MID.UnknownCommand, command);
660   }
661   Stdout(msg).newline;
662 }
663 
664 void runUnittests()
665 {
666   import cmd.ImportGraph,
667          dil.ast.Visitor,
668          dil.doc.Macro,
669          dil.doc.Parser,
670          dil.lexer.Lexer,
671          dil.Float,
672          dil.Complex,
673          dil.Converter,
674          dil.FileBOM,
675          dil.HtmlEntities,
676          dil.String,
677          util.uni;
678 
679   auto testFuncs = [
680     &testGraph, &testFloat, &testComplex, &testConverter,
681     &testTellBOM, &testEntity2Unicode, &testString,
682     &testVisitor, &testMacroConvert, &testDocParser,
683     &testLexer, &testLexerPeek, &testIsUniAlpha,
684     &testSet,
685   ];
686 
687   foreach (testFunc; testFuncs)
688     testFunc();
689 }