dsnapshot.layout_utils source code

1 /**
2 Copyright: Copyright (c) 2019, Joakim Brännström. All rights reserved.
3 License: $(LINK2 http://www.boost.org/LICENSE_1_0.txt, Boost Software License 1.0)
4 Author: Joakim Brännström (joakim.brannstrom@gmx.com)
5 */
6 module dsnapshot.layout_utils;
7 
8 import logger = std.experimental.logger;
9 import std.array : empty;
10 import std.exception : collectException;
11 
12 import sumtype;
13 
14 import dsnapshot.exception;
15 import dsnapshot.layout : Name, Layout;
16 import dsnapshot.types;
17 
18 version (unittest) {
19     import unit_threaded.assertions;
20 }
21 
22 @safe:
23 
24 Layout fillLayout(const Layout layout_, const Flow flow, const RemoteCmd cmd) {
25     import std.algorithm : filter, map, sort;
26     import std.array : array;
27     import std.conv : to;
28     import std.datetime : UTC, DateTimeException, SysTime;
29     import std.file : dirEntries, SpanMode, exists, isDir;
30     import std.path : baseName;
31     import dsnapshot.layout : LSnapshot = Snapshot;
32 
33     Layout rval = layout_.dup;
34 
35     const names = flow.match!((None a) => null,
36             (FlowRsyncToLocal a) => snapshotNamesFromDir(a.dst.value.Path),
37             (FlowLocal a) => snapshotNamesFromDir(a.dst.value.Path),
38             (FlowLocalToRsync a) => snapshotNamesFromSsh(cmd, a.dst.addr, a.dst.path));
39 
40     foreach (const n; names) {
41         try {
42             const t = SysTime.fromISOExtString(n.value, UTC());
43             rval.put(LSnapshot(t, n));
44         } catch (DateTimeException e) {
45             logger.warning("Unable to extract the time from the snapshot name");
46             logger.info(e.msg);
47             logger.info("It is added as a snapshot taken at the time ", SysTime.min);
48             rval.put(LSnapshot(SysTime.min, n));
49         }
50     }
51 
52     rval.finalize;
53 
54     return rval;
55 }
56 
57 Name[] snapshotNamesFromDir(Path dir) @trusted {
58     import std.algorithm : filter, map, copy;
59     import std.array : appender;
60     import std.file : dirEntries, SpanMode, exists, isDir;
61     import std.path : baseName;
62 
63     if (!dir.toString.exists)
64         return null;
65     if (!dir.toString.isDir)
66         throw new SnapshotException(SnapshotException.DstIsNotADir.init.SnapshotError);
67 
68     auto app = appender!(Name[])();
69     dirEntries(dir.toString, SpanMode.shallow).map!(a => a.name)
70         .filter!(a => a.isDir)
71         .map!(a => Name(a.baseName))
72         .copy(app);
73     return app.data;
74 }
75 
76 Name[] snapshotNamesFromSsh(const RemoteCmd cmd_, string addr, string path) {
77     import std.algorithm : map, copy;
78     import std.array : appender;
79     import std.process : execute;
80     import std.string : splitLines;
81 
82     auto cmd = cmd_.match!((const SshRemoteCmd a) {
83         return a.toCmd(RemoteSubCmd.lsDirs, addr, path);
84     });
85     if (cmd.empty)
86         return null;
87 
88     auto res = execute(cmd);
89     if (res.status != 0) {
90         logger.warning(res.output);
91         return null;
92     }
93 
94     auto app = appender!(Name[])();
95     res.output.splitLines.map!(a => Name(a)).copy(app);
96 
97     return app.data;
98 }
99 
100 @("shall scan the directory for all snapshots")
101 unittest {
102     import std.conv : to;
103     import std.datetime : SysTime, UTC, Duration, Clock, dur, Interval;
104     import std.file;
105     import std.path;
106     import std.range : enumerate;
107     import sumtype;
108     import dsnapshot.layout : Layout, LayoutConfig, Span;
109 
110     immutable tmpDir = "test_snapshot_scan";
111     scope (exit)
112         () @trusted { rmdirRecurse(tmpDir); }();
113     mkdir(tmpDir).collectException;
114 
115     const offset = 5.dur!"minutes";
116     const base = Clock.currTime.toUTC;
117     auto curr = cast() base;
118 
119     foreach (const i; 0 .. 15) {
120         mkdir(buildPath(tmpDir, curr.toISOExtString));
121         curr -= 4.dur!"hours";
122     }
123     foreach (const i; 0 .. 4) {
124         curr -= 8.dur!"hours";
125         mkdir(buildPath(tmpDir, curr.toISOExtString));
126     }
127 
128     auto conf = LayoutConfig([Span(5, 4.dur!"hours"), Span(5, 1.dur!"days")]);
129     auto layout = Layout(base, conf);
130     layout = fillLayout(layout, FlowLocal(LocalAddr(tmpDir), LocalAddr(tmpDir))
131             .Flow, RemoteCmd(SshRemoteCmd.init));
132 
133     layout.discarded.length.shouldEqual(10);
134 
135     (base - layout.snapshotTimeInBucket(0).get).total!"minutes".shouldEqual(0);
136     (base - layout.snapshotTimeInBucket(4).get).total!"hours".shouldEqual(3 * 5 + 1);
137     (base - layout.snapshotTimeInBucket(5).get).total!"hours".shouldEqual(4 * 5);
138     (base - layout.snapshotTimeInBucket(6).get).total!"hours".shouldEqual(4 * 5 + 24 * 1);
139     (base - layout.snapshotTimeInBucket(7).get).total!"hours".shouldEqual(4 * 5 + 24 * 2);
140     /// these buckets are filled by the second  pass
141     (base - layout.snapshotTimeInBucket(8).get).total!"hours".shouldEqual(4 * 5 + 24 * 3);
142     layout.snapshotTimeInBucket(9).isNull.shouldBeTrue;
143 }