watchexec source code

1 /**
2 Copyright: Copyright (c) 2020, Joakim Brännström. All rights reserved.
3 License: $(LINK2 http://www.boost.org/LICENSE_1_0.txt, Boost Software License 1.0)
4 Author: Joakim Brännström (joakim.brannstrom@gmx.com)
5 */
6 module watchexec;
7 
8 import core.thread : Thread;
9 import logger = std.experimental.logger;
10 import std.algorithm : filter, map, joiner;
11 import std.array : array, empty, appender;
12 import std.conv : text;
13 import std.datetime : Duration;
14 import std.datetime : dur, Clock;
15 import std.exception : collectException;
16 import std.file : readLink;
17 import std.format : format;
18 
19 import colorlog;
20 import my.file : existsAnd, isSymlink;
21 import my.filter : GlobFilter;
22 import my.filter;
23 import my.fswatch : Monitor, MonitorResult;
24 import my.optional;
25 import my.path;
26 
27 version (unittest) {
28 } else {
29     int main(string[] args) {
30         confLogger(VerboseMode.info);
31 
32         auto conf = parseUserArgs(args);
33 
34         confLogger(conf.global.verbosity);
35         logger.trace(conf);
36 
37         if (conf.global.help)
38             return cliHelp(conf);
39         return cli(conf);
40     }
41 }
42 
43 private:
44 
45 immutable notifySendCmd = "notify-send";
46 
47 immutable defaultExclude = [
48     "*/.DS_Store", "*.py[co]", "*/#*#", "*/.#*", "*/.*.kate-swp", "*/.*.sw?",
49     "*/.*.sw?x", "*/.git/*"
50 ];
51 
52 int cliHelp(AppConfig conf) {
53     conf.printHelp;
54     return 0;
55 }
56 
57 int cli(AppConfig conf) {
58     import std.process : userShell;
59     import std.stdio : write, writeln;
60     import my.fswatch : ContentEvents, MetadataEvents, FileWatch;
61     import proc;
62 
63     if (conf.global.paths.empty) {
64         logger.error("No directories specified to watch");
65         return 1;
66     }
67     if (conf.global.command.empty) {
68         logger.error("No command to execute specified");
69         return 1;
70     }
71 
72     auto defaultFilter = GlobFilter(conf.global.include, conf.global.exclude);
73     auto recurseFilter = conf.global.useVcsIgnore
74         ? parseGitIgnoreRecursive(conf.global.include, conf.global.paths) : null;
75 
76     logger.infof("command to execute on change: %-(%s %)", conf.global.command);
77 
78     auto cmd = () {
79         if (conf.global.useShell)
80             logger.info("--shell is deprecated");
81         return [userShell, "-c", format!"%-(%s %)"(conf.global.command)];
82     }();
83 
84     auto handleExitStatus = HandleExitStatus(conf.global.useNotifySend);
85 
86     if (conf.global.oneShotMode)
87         return cliOneshot(conf, cmd, handleExitStatus, defaultFilter);
88 
89     logger.infof("watching: %s", conf.global.paths);
90     checkPaths(conf.global.paths);
91     logger.info("starting");
92 
93     auto monitor = Monitor(conf.global.paths, defaultFilter, recurseFilter,
94             FileWatch.FollowSymlink(conf.global.followSymlink),
95             conf.global.watchMetadata ? (ContentEvents | MetadataEvents) : ContentEvents);
96 
97     MonitorResult[] buildAndExecute(MonitorResult[] eventFiles) {
98         string[string] env;
99         MonitorResult[] rval;
100 
101         if (conf.global.setEnv) {
102             env["WATCHEXEC_EVENT"] = eventFiles.map!(a => format!"%s:%s"(a.kind,
103                     a.path)).joiner(";").text;
104         }
105 
106         auto p = spawnProcess(cmd, env).sandbox.timeout(conf.global.timeout)
107             .rcKill(conf.global.signal);
108 
109         if (conf.global.restart) {
110             while (!p.tryWait && rval.empty) {
111                 rval = monitor.wait(10.dur!"msecs");
112             }
113 
114             if (rval.empty) {
115                 handleExitStatus.exitStatus(p.status);
116             } else {
117                 p.kill(conf.global.signal);
118                 p.wait;
119                 logger.info("restarting command".color(Color.yellow));
120             }
121         } else {
122             p.wait;
123             handleExitStatus.exitStatus(p.status);
124         }
125 
126         if (conf.global.clearEvents) {
127             // the events can fire a bit late when e.g. writing to an NFS mount
128             // point.
129             monitor.collect(10.dur!"msecs");
130         }
131 
132         return rval;
133     }
134 
135     MonitorResult[] eventFiles;
136 
137     if (!conf.global.postPone) {
138         try {
139             eventFiles = buildAndExecute(null);
140         } catch (Exception e) {
141             logger.error(e.msg);
142             return 1;
143         }
144     }
145 
146     while (true) {
147         if (eventFiles.empty) {
148             eventFiles = monitor.wait(1000.dur!"weeks");
149         }
150 
151         foreach (changed; eventFiles) {
152             logger.tracef("%s changed", changed);
153         }
154 
155         if (!eventFiles.empty) {
156             if (conf.global.debounce != Duration.zero) {
157                 eventFiles ~= monitor.collect(conf.global.debounce);
158             }
159 
160             if (conf.global.clearScreen) {
161                 write("\033c");
162             }
163 
164             try {
165                 eventFiles = buildAndExecute(eventFiles);
166             } catch (Exception e) {
167                 logger.error(e.msg);
168                 return 1;
169             }
170 
171         }
172     }
173 }
174 
175 int cliOneshot(AppConfig conf, string[] cmd, HandleExitStatus handleExitStatus,
176         GlobFilter defaultFilter) {
177     import std.algorithm : map, filter, joiner, cache;
178     import std.datetime : Clock;
179     import std.file : exists, readText, isDir, isFile, timeLastModified, getSize, rename;
180     import std.parallelism : taskPool, task;
181     import std.stdio : File;
182     import std.typecons : tuple, Tuple;
183     import my.file : existsAnd, followSymlink;
184     import my.optional;
185     import watchexec_internal.oneshot;
186 
187     static bool update(ref FileDb db, ref FileDb newDb, ref OneShotFile f) {
188         const changed = db.isChanged(f);
189 
190         if (changed)
191             newDb.add(f);
192         else if (auto v = f.path in db.files)
193             newDb.add(*v); // avoid checksum calculation
194         else
195             newDb.add(f);
196         return changed;
197     }
198 
199     auto envApp = appender!(string[])();
200     void updateEnv(AbsolutePath p, MonitorResult.Kind kind) {
201         if (!conf.global.setEnv)
202             return;
203         envApp.put(format!"%s:%s"(kind, p.toString));
204     }
205 
206     auto db = () {
207         if (!exists(conf.global.jsonDb))
208             return FileDb.init;
209 
210         try {
211             logger.infof("Reading %s", conf.global.jsonDb);
212             return fromJson(readText(conf.global.jsonDb));
213         } catch (Exception e) {
214         }
215 
216         return FileDb.init;
217     }();
218 
219     bool isChanged;
220     FileDb newDb;
221     newDb.command = cmd;
222 
223     try {
224         foreach (f; conf.global
225                 .paths
226                 .filter!(existsAnd!isDir)
227                 .map!(a => OneShotRange(a, defaultFilter, conf.global.followSymlink))
228                 .joiner
229                 .filter!(a => a.hasValue)
230                 .map!(a => a.orElse(OneShotFile.init))) {
231             const changed = update(db, newDb, f);
232             isChanged = isChanged || changed;
233             if (changed)
234                 updateEnv(f.path, MonitorResult.Kind.Modify);
235         }
236 
237         foreach (fname; conf.global.paths.filter!(existsAnd!isFile)) {
238             auto f = OneShotFile(AbsolutePath(fname), TimeStamp(timeLastModified(fname.toString,
239                     Clock.currTime).toUnixTime), FileSize(getSize(fname)));
240             const changed = update(db, newDb, f);
241             isChanged = isChanged || changed;
242             if (changed)
243                 updateEnv(f.path, MonitorResult.Kind.Modify);
244         }
245 
246         // find deleted files
247         foreach (a; db.files.byKey.filter!(a => a !in newDb.files)) {
248             isChanged = true;
249             updateEnv(a, MonitorResult.Kind.Delete);
250             break;
251         }
252 
253         isChanged = isChanged || db.command != cmd;
254     } catch (Exception e) {
255         logger.info(e.msg).collectException;
256     }
257 
258     int exitStatus;
259     if (isChanged) {
260         import proc;
261 
262         const tmpDb = AbsolutePath(conf.global.jsonDb ~ ".tmp");
263         auto saveDb = task(() {
264             try {
265                 File(tmpDb, "w").write(toJson(newDb));
266             } catch (Exception e) {
267                 logger.info(e.msg).collectException;
268             }
269         });
270         saveDb.executeInNewThread;
271 
272         auto env = () {
273             string[string] env;
274             if (conf.global.setEnv)
275                 env["WATCHEXEC_EVENT"] = envApp.data.joiner(";").text;
276             return env;
277         }();
278         auto p = spawnProcess(cmd, env).sandbox.timeout(conf.global.timeout)
279             .rcKill(conf.global.signal);
280         p.wait;
281         exitStatus = p.status;
282 
283         saveDb.yieldForce;
284 
285         if (exitStatus == 0)
286             try {
287                 rename(tmpDb, conf.global.jsonDb);
288             } catch (Exception e) {
289                 logger.warning(e.msg);
290             }
291 
292         logger.trace("old database: ", db);
293         logger.trace("new database: ", newDb);
294     }
295 
296     handleExitStatus.exitStatus(exitStatus);
297     return exitStatus;
298 }
299 
300 // Check if the paths exists and if it doesn't print a warning to the user.
301 // For now I think a warning is the best case because watchexec do support
302 // watching for files that do not yet exist.
303 void checkPaths(AbsolutePath[] paths) nothrow {
304     import std.file : exists;
305 
306     foreach (p; paths) {
307         try {
308             if (!exists(p.toString))
309                 logger.warning("path do not exist: ", p);
310         } catch (Exception e) {
311             logger.warning(e.msg).collectException;
312         }
313     }
314 }
315 
316 struct HandleExitStatus {
317     bool useNotifySend;
318     string notifyMsg;
319 
320     this(string notifyMsg) {
321         this.useNotifySend = !notifyMsg.empty;
322         this.notifyMsg = notifyMsg;
323     }
324 
325     void exitStatus(int code) {
326         import std.conv : to;
327         import std.process : spawnProcess, wait;
328 
329         immutable msgExitStatus = "exit status";
330         immutable msgOk = "✓";
331         immutable msgNok = "✗";
332 
333         if (useNotifySend) {
334             auto msg = () {
335                 if (code == 0)
336                     return format!"%s %s %s\n%s"(msgOk, msgExitStatus, code, notifyMsg);
337                 return format!"%s %s %s\n%s"(msgNok, msgExitStatus, code, notifyMsg);
338             }();
339 
340             spawnProcess([
341                     notifySendCmd, "-u", "normal", "-t", "3000", "-a", "watchexec",
342                     msg
343                     ]).wait;
344         }
345 
346         auto msg = () {
347             if (code == 0)
348                 return format!"%s %s"(msgOk, msgExitStatus.color(Color.green));
349             return format!"%s %s"(msgNok, msgExitStatus.color(Color.red));
350         }();
351 
352         logger.infof("%s %s", msg, code);
353     }
354 }
355 
356 struct AppConfig {
357     static import std.getopt;
358 
359     static struct Global {
360         import core.sys.posix.signal : SIGKILL;
361 
362         std.getopt.GetoptResult helpInfo;
363         VerboseMode verbosity;
364         bool help = true;
365 
366         AbsolutePath[] paths;
367         Duration debounce;
368         Duration timeout;
369         bool clearEvents;
370         bool clearScreen;
371         bool followSymlink;
372         bool oneShotMode;
373         bool postPone;
374         bool restart;
375         bool setEnv;
376         bool useShell;
377         bool useVcsIgnore;
378         bool watchMetadata;
379         int signal = SIGKILL;
380         string jsonDb = "watchexec_db.json";
381         string progName;
382         string useNotifySend;
383         string[] command;
384 
385         string[] include;
386         string[] exclude;
387     }
388 
389     Global global;
390 
391     void printHelp() {
392         std.getopt.defaultGetoptPrinter(format(
393                 "Execute commands when watched files change\nusage: %s [options] -- <command>\n\noptions:",
394                 global.progName), global.helpInfo.options);
395     }
396 }
397 
398 AppConfig parseUserArgs(string[] args) {
399     import logger = std.experimental.logger;
400     import std.algorithm : countUntil, map;
401     import std.path : baseName;
402     import std.traits : EnumMembers;
403     import my.file : existsAnd, isFile, whichFromEnv, followSymlink;
404     static import std.getopt;
405 
406     AppConfig conf;
407     conf.global.progName = args[0].baseName;
408 
409     try {
410         const idx = countUntil(args, "--");
411         if (args.length > 1 && idx > 1) {
412             conf.global.command = args[idx + 1 .. $];
413             args = args[0 .. idx];
414         }
415 
416         bool clearEvents;
417         bool noDefaultIgnore;
418         bool noVcsIgnore;
419         bool noFollowSymlink;
420         string[] include;
421         string[] monitorExtensions;
422         string[] paths;
423         uint debounce = 200;
424         uint timeout = 3600;
425         // dfmt off
426         conf.global.helpInfo = std.getopt.getopt(args,
427             "clear-events", "clear the events that occured when executing the command", &clearEvents,
428             "c|clear", "clear screen before executing command",&conf.global.clearScreen,
429             "d|debounce", format!"set the timeout between detected change and command execution (default: %sms)"(debounce), &debounce,
430             "env", "set WATCHEXEC_EVENT environment variables when executing the command", &conf.global.setEnv,
431             "exclude", "ignore modifications to paths matching the pattern (glob: <empty>)", &conf.global.exclude,
432             "e|ext", "file extensions, excluding dot, to watch (default: any)", &monitorExtensions,
433             "include", "ignore all modifications except those matching the pattern (glob: *)", &conf.global.include,
434             "meta", "watch for metadata changes (date, open/close, permission)", &conf.global.watchMetadata,
435             "no-default-ignore", "skip auto-ignoring of commonly ignored globs", &noDefaultIgnore,
436             "no-follow-symlink", "do not follow symlinks", &noFollowSymlink,
437             "no-vcs-ignore", "skip auto-loading of .gitignore files for filtering", &noVcsIgnore,
438             "notify", format!"use %s for desktop notification with commands exit status and this msg"(notifySendCmd), &conf.global.useNotifySend,
439             "oneshot-db", "json database to use", &conf.global.jsonDb,
440             "o|oneshot", "run in one-shot mode where the command is executed if files are different from watchexec.json", &conf.global.oneShotMode,
441             "p|postpone", "wait until first change to execute command", &conf.global.postPone,
442             "r|restart", "restart the process if it's still running", &conf.global.restart,
443             "shell", "(deprecated) run the command in a shell (/bin/sh)", &conf.global.useShell,
444             "s|signal", "send signal to process upon changes, e.g. SIGHUP (default: SIGKILL)", &conf.global.signal,
445             "t|timeout", format!"max runtime of the command (default: %ss)"(timeout), &timeout,
446             "v|verbose", format("Set the verbosity (%-(%s, %))", [EnumMembers!(VerboseMode)]), &conf.global.verbosity,
447             "w|watch", "watch a specific directory", &paths,
448             );
449         // dfmt on
450 
451         conf.global.clearEvents = clearEvents;
452         conf.global.followSymlink = !noFollowSymlink;
453 
454         include ~= monitorExtensions.map!(a => format!"*.%s"(a)).array;
455         if (include.empty) {
456             conf.global.include = ["*"];
457         } else {
458             conf.global.include = include;
459         }
460 
461         if (!noVcsIgnore && existsAnd!isFile(Path(".gitignore"))) {
462             import std.file : readText;
463 
464             try {
465                 conf.global.exclude ~= parseGitIgnore(readText(".gitignore"));
466             } catch (Exception e) {
467                 logger.warning(e.msg);
468             }
469         } else if (!noDefaultIgnore && !noVcsIgnore) {
470             conf.global.exclude ~= defaultExclude;
471         }
472         conf.global.useVcsIgnore = !noVcsIgnore;
473 
474         if (conf.global.useNotifySend) {
475             if (!whichFromEnv("PATH", notifySendCmd)) {
476                 conf.global.useNotifySend = null;
477                 logger.warningf("--notify requires the command %s", notifySendCmd);
478             }
479         }
480 
481         conf.global.timeout = timeout.dur!"seconds";
482         conf.global.debounce = debounce.dur!"msecs";
483         conf.global.paths = () {
484             return (conf.global.followSymlink) ? (paths.map!(a => AbsolutePath(a)).array) : (
485                     paths.map!(a => followSymlink(Path(a)).orElse(Path.init))
486                     .filter!(a => !a.empty)
487                     .map!(a => AbsolutePath(a))
488                     .array);
489         }();
490 
491         conf.global.help = conf.global.helpInfo.helpWanted;
492     } catch (std.getopt.GetOptException e) {
493         // unknown option
494         logger.error(e.msg);
495     } catch (Exception e) {
496         logger.error(e.msg);
497     }
498 
499     return conf;
500 }
501 
502 /** Returns: the glob patterns from `content`.
503  *
504  * The syntax is the one found in .gitignore files.
505  */
506 string[] parseGitIgnore(string content) {
507     import std.algorithm : splitter;
508     import std.array : appender;
509     import std.ascii : newline;
510     import std.string : strip;
511 
512     auto app = appender!(string[])();
513 
514     foreach (l; content.splitter(newline).filter!(a => !a.empty)
515             .filter!(a => a[0] != '#')) {
516         app.put(l.strip);
517     }
518 
519     return app.data;
520 }
521 
522 @("shall parse a file with gitignore syntax")
523 unittest {
524     auto res = parseGitIgnore(`*.[oa]
525 *.obj
526 *.svn
527 
528 # editor junk files
529 *~
530 *.orig
531 tags
532 *.swp
533 
534 # dlang
535 build/
536 .dub
537 docs.json
538 __dummy.html
539 *.lst
540 __test__*__
541 
542 # rust
543 target/
544 **/*.rs.bk
545 
546 # python
547 *.pyc
548 
549 repo.tar.gz`);
550     assert(res == [
551             "*.[oa]", "*.obj", "*.svn", "*~", "*.orig", "tags", "*.swp", "build/",
552             ".dub", "docs.json", "__dummy.html", "*.lst", "__test__*__",
553             "target/", "**/*.rs.bk", "*.pyc", "repo.tar.gz"
554             ]);
555 }
556 
557 GlobFilter[AbsolutePath] parseGitIgnoreRecursive(string[] includes, AbsolutePath[] roots) {
558     import std.array : appender;
559     import std.file : dirEntries, SpanMode, isFile, readText, isDir;
560     import std.path : baseName, dirName;
561     import my.file : existsAnd;
562 
563     GlobFilter[AbsolutePath] rval;
564 
565     foreach (gf; roots.filter!(existsAnd!isDir)
566             .map!(a => dirEntries(a.toString, SpanMode.depth))
567             .joiner
568             .map!(a => Path(a.name))
569             .filter!(existsAnd!isFile)
570             .filter!(a => a.baseName == ".gitignore")) {
571         try {
572             rval[AbsolutePath(gf.dirName)] = GlobFilter(includes, parseGitIgnore(readText(gf)));
573         } catch (Exception e) {
574         }
575     }
576 
577     return rval;
578 }