1 /**
2 Copyright: Copyright (c) 2019, Joakim Brännström. All rights reserved.
3 License: $(LINK2 http://www.boost.org/LICENSE_1_0.txt, Boost Software License 1.0)
4 Author: Joakim Brännström (joakim.brannstrom@gmx.com)
5 */
6 module dsnapshot.layout_utils;
7 
8 import logger = std.experimental.logger;
9 import std.array : empty;
10 import std.exception : collectException;
11 
12 import sumtype;
13 
14 import dsnapshot.exception;
15 import dsnapshot.layout : Name, Layout;
16 import dsnapshot.types;
17 
18 version (unittest) {
19     import unit_threaded.assertions;
20 }
21 
22 @safe:
23 
24 Layout fillLayout(const Layout layout_, const Flow flow, const RemoteCmd cmd) {
25     import std.algorithm : filter, map, sort;
26     import std.array : array;
27     import std.conv : to;
28     import std.datetime : UTC, DateTimeException, SysTime;
29     import std.file : dirEntries, SpanMode, exists, isDir;
30     import std.path : baseName;
31     import dsnapshot.layout : LSnapshot = Snapshot;
32 
33     Layout rval = layout_.dup;
34 
35     const names = flow.match!((None a) => null,
36             (FlowRsyncToLocal a) => snapshotNamesFromDir(a.dst.value.Path),
37             (FlowLocal a) => snapshotNamesFromDir(a.dst.value.Path),
38             (FlowLocalToRsync a) => snapshotNamesFromSsh(cmd, a.dst.addr, a.dst.path));
39 
40     foreach (const n; names) {
41         try {
42             const t = SysTime.fromISOExtString(n.value, UTC());
43             rval.put(LSnapshot(t, n));
44         } catch (DateTimeException e) {
45             logger.warning("Unable to extract the time from the snapshot name");
46             logger.info(e.msg);
47             logger.info("It is added as a snapshot taken at the time ", SysTime.min);
48             rval.put(LSnapshot(SysTime.min, n));
49         }
50     }
51 
52     rval.finalize;
53 
54     return rval;
55 }
56 
57 Name[] snapshotNamesFromDir(Path dir) @trusted {
58     import std.algorithm : filter, map, copy;
59     import std.array : appender;
60     import std.file : dirEntries, SpanMode, exists, isDir;
61     import std.path : baseName;
62 
63     if (!dir.toString.exists)
64         return null;
65     if (!dir.toString.isDir)
66         throw new SnapshotException(SnapshotException.DstIsNotADir.init.SnapshotError);
67 
68     auto app = appender!(Name[])();
69     dirEntries(dir.toString, SpanMode.shallow).map!(a => a.name)
70         .filter!(a => a.isDir)
71         .filter!(a => a.baseName != snapshotLatest)
72         .map!(a => Name(a.baseName))
73         .copy(app);
74     return app.data;
75 }
76 
77 Name[] snapshotNamesFromSsh(const RemoteCmd cmd_, string addr, string path) {
78     import std.algorithm : map, copy, filter;
79     import std.array : appender;
80     import std.path : baseName;
81     import std.process : execute;
82     import std.string : splitLines;
83 
84     auto cmd = cmd_.match!((const SshRemoteCmd a) {
85         return a.toCmd(RemoteSubCmd.lsDirs, addr, path);
86     });
87     if (cmd.empty)
88         return null;
89 
90     auto res = execute(cmd);
91     if (res.status != 0) {
92         logger.warning(res.output);
93         return null;
94     }
95 
96     auto app = appender!(Name[])();
97     res.output
98         .splitLines
99         .filter!(a => a.baseName != snapshotLatest)
100         .map!(a => Name(a))
101         .copy(app);
102 
103     return app.data;
104 }
105 
106 @("shall scan the directory for all snapshots")
107 unittest {
108     import std.conv : to;
109     import std.datetime : SysTime, UTC, Duration, Clock, dur, Interval;
110     import std.file;
111     import std.path;
112     import std.range : enumerate;
113     import sumtype;
114     import dsnapshot.layout : Layout, LayoutConfig, Span;
115 
116     immutable tmpDir = "test_snapshot_scan";
117     scope (exit)
118         () @trusted { rmdirRecurse(tmpDir); }();
119     () @trusted { rmdirRecurse(tmpDir).collectException; }();
120     mkdir(tmpDir).collectException;
121 
122     const offset = 5.dur!"minutes";
123     const base = Clock.currTime.toUTC;
124     auto curr = cast() base;
125 
126     foreach (const i; 0 .. 15) {
127         mkdir(buildPath(tmpDir, curr.toISOExtString));
128         curr -= 4.dur!"hours";
129     }
130     foreach (const i; 0 .. 4) {
131         curr -= 8.dur!"hours";
132         mkdir(buildPath(tmpDir, curr.toISOExtString));
133     }
134 
135     auto conf = LayoutConfig([Span(5, 4.dur!"hours"), Span(5, 1.dur!"days")]);
136     auto layout = Layout(base, conf);
137     layout = fillLayout(layout, FlowLocal(LocalAddr(tmpDir), LocalAddr(tmpDir))
138             .Flow, RemoteCmd(SshRemoteCmd.init));
139 
140     layout.discarded.length.shouldEqual(10);
141 
142     (base - layout.snapshotTimeInBucket(0).get).total!"minutes".shouldEqual(0);
143     (base - layout.snapshotTimeInBucket(4).get).total!"hours".shouldEqual(3 * 5 + 1);
144     (base - layout.snapshotTimeInBucket(5).get).total!"hours".shouldEqual(4 * 5);
145     (base - layout.snapshotTimeInBucket(6).get).total!"hours".shouldEqual(4 * 5 + 24 * 1);
146     (base - layout.snapshotTimeInBucket(7).get).total!"hours".shouldEqual(4 * 5 + 24 * 2);
147     /// these buckets are filled by the second  pass
148     (base - layout.snapshotTimeInBucket(8).get).total!"hours".shouldEqual(4 * 5 + 24 * 3);
149     layout.snapshotTimeInBucket(9).isNull.shouldBeTrue;
150 }