1 /**
2 Copyright: Copyright (c) 2019, Joakim Brännström. All rights reserved.
3 License: $(LINK2 http://www.boost.org/LICENSE_1_0.txt, Boost Software License 1.0)
4 Author: Joakim Brännström (joakim.brannstrom@gmx.com)
5 */
6 module dsnapshot.layout_utils;
7 
8 import logger = std.experimental.logger;
9 import std.array : empty;
10 import std.exception : collectException;
11 
12 import sumtype;
13 
14 import dsnapshot.exception;
15 import dsnapshot.layout : Name, Layout;
16 import dsnapshot.types;
17 
18 version (unittest) {
19     import unit_threaded.assertions;
20 }
21 
22 auto fillLayout(Layout layout_, Flow flow, const RemoteCmd cmd) {
23     import std.algorithm : filter, map, sort;
24     import std.array : array;
25     import std.conv : to;
26     import std.datetime : UTC, DateTimeException, SysTime;
27     import std.file : dirEntries, SpanMode, exists, isDir;
28     import std.path : baseName;
29     import dsnapshot.layout : LSnapshot = Snapshot;
30 
31     auto rval = layout_;
32     scope (exit)
33         rval.finalize;
34 
35     const names = flow.match!((None a) => null,
36             (FlowRsyncToLocal a) => snapshotNamesFromDir(a.dst.value.Path),
37             (FlowLocal a) => snapshotNamesFromDir(a.dst.value.Path),
38             (FlowLocalToRsync a) => snapshotNamesFromSsh(cmd, a.dst.addr, a.dst.path));
39 
40     foreach (const n; names) {
41         try {
42             const t = SysTime.fromISOExtString(n.value, UTC());
43             rval.put(LSnapshot(t, n));
44         } catch (DateTimeException e) {
45             logger.warning("Unable to extract the time from the snapshot name");
46             logger.info(e.msg);
47             logger.info("It is added as a snapshot taken at the time ", SysTime.min);
48             rval.put(LSnapshot(SysTime.min, n));
49         }
50     }
51 
52     return rval;
53 }
54 
55 Name[] snapshotNamesFromDir(Path dir) {
56     import std.algorithm : filter, map, copy;
57     import std.array : appender;
58     import std.file : dirEntries, SpanMode, exists, isDir;
59     import std.path : baseName;
60 
61     if (!dir.toString.exists)
62         return null;
63     if (!dir.toString.isDir)
64         throw new SnapshotException(SnapshotException.DstIsNotADir.init.SnapshotError);
65 
66     auto app = appender!(Name[])();
67     dirEntries(dir.toString, SpanMode.shallow).map!(a => a.name)
68         .filter!(a => a.isDir)
69         .map!(a => Name(a.baseName))
70         .copy(app);
71     return app.data;
72 }
73 
74 Name[] snapshotNamesFromSsh(const RemoteCmd cmd_, string addr, string path) {
75     import std.algorithm : map, copy;
76     import std.array : appender;
77     import std.process : execute;
78     import std..string : splitLines;
79 
80     auto cmd = cmd_.match!((const SshRemoteCmd a) {
81         return a.toCmd(RemoteSubCmd.lsDirs, addr, path);
82     });
83     if (cmd.empty)
84         return null;
85 
86     auto res = execute(cmd);
87     if (res.status != 0) {
88         logger.warning(res.output);
89         return null;
90     }
91 
92     auto app = appender!(Name[])();
93     res.output.splitLines.map!(a => Name(a)).copy(app);
94 
95     return app.data;
96 }
97 
98 @("shall scan the directory for all snapshots")
99 unittest {
100     import std.conv : to;
101     import std.datetime : SysTime, UTC, Duration, Clock, dur;
102     import std.file;
103     import std.path;
104     import std.range : enumerate;
105     import sumtype;
106     import dsnapshot.layout : Layout, LayoutConfig, Span;
107 
108     immutable tmpDir = "test_snapshot_scan";
109     scope (exit)
110         rmdirRecurse(tmpDir);
111     mkdir(tmpDir);
112 
113     auto curr = Clock.currTime;
114     curr.timezone = UTC();
115 
116     foreach (const i; 0 .. 15) {
117         mkdir(buildPath(tmpDir, curr.toISOExtString));
118         curr -= 1.dur!"hours";
119     }
120     foreach (const i; 0 .. 15) {
121         curr -= 5.dur!"hours";
122         mkdir(buildPath(tmpDir, curr.toISOExtString));
123     }
124 
125     auto conf = LayoutConfig([Span(5, 4.dur!"hours"), Span(5, 1.dur!"days")]);
126     const base = Clock.currTime;
127     auto layout = Layout(base, conf);
128     layout = fillLayout(layout, FlowLocal(LocalAddr(tmpDir), LocalAddr(tmpDir))
129             .Flow, RemoteCmd(SshRemoteCmd.init));
130 
131     layout.waiting.length.shouldEqual(0);
132     layout.discarded.length.shouldEqual(20);
133 
134     (base - layout.snapshotTimeInBucket(0).get).total!"hours".shouldEqual(4);
135     (base - layout.snapshotTimeInBucket(4).get).total!"hours".shouldEqual(4 * 5);
136     (base - layout.snapshotTimeInBucket(5).get).total!"hours".shouldEqual(4 * 5 + 24 + 1);
137     (base - layout.snapshotTimeInBucket(6).get).total!"hours".shouldEqual(4 * 5 + 24 * 2 + 2);
138     (base - layout.snapshotTimeInBucket(7).get).total!"hours".shouldEqual(4 * 5 + 24 * 3 - 2);
139 
140     /// these buckets are filled by the second  pass
141     (base - layout.snapshotTimeInBucket(8).get).total!"hours".shouldEqual(4 * 5 + 24 * 4 - 31);
142     (base - layout.snapshotTimeInBucket(9).get).total!"hours".shouldEqual(4 * 5 + 24 * 5 - 60);
143 }