1 /** 2 Copyright: Copyright (c) 2019, Joakim Brännström. All rights reserved. 3 License: $(LINK2 http://www.boost.org/LICENSE_1_0.txt, Boost Software License 1.0) 4 Author: Joakim Brännström (joakim.brannstrom@gmx.com) 5 */ 6 module dsnapshot.layout_utils; 7 8 import logger = std.experimental.logger; 9 import std.array : empty; 10 import std.exception : collectException; 11 12 import sumtype; 13 14 import dsnapshot.exception; 15 import dsnapshot.layout : Name, Layout; 16 import dsnapshot.types; 17 18 version (unittest) { 19 import unit_threaded.assertions; 20 } 21 22 auto fillLayout(Layout layout_, Flow flow, const RemoteCmd cmd) { 23 import std.algorithm : filter, map, sort; 24 import std.array : array; 25 import std.conv : to; 26 import std.datetime : UTC, DateTimeException, SysTime; 27 import std.file : dirEntries, SpanMode, exists, isDir; 28 import std.path : baseName; 29 import dsnapshot.layout : LSnapshot = Snapshot; 30 31 auto rval = layout_; 32 scope (exit) 33 rval.finalize; 34 35 const names = flow.match!((None a) => null, 36 (FlowRsyncToLocal a) => snapshotNamesFromDir(a.dst.value.Path), 37 (FlowLocal a) => snapshotNamesFromDir(a.dst.value.Path), 38 (FlowLocalToRsync a) => snapshotNamesFromSsh(cmd, a.dst.addr, a.dst.path)); 39 40 foreach (const n; names) { 41 try { 42 const t = SysTime.fromISOExtString(n.value, UTC()); 43 rval.put(LSnapshot(t, n)); 44 } catch (DateTimeException e) { 45 logger.warning("Unable to extract the time from the snapshot name"); 46 logger.info(e.msg); 47 logger.info("It is added as a snapshot taken at the time ", SysTime.min); 48 rval.put(LSnapshot(SysTime.min, n)); 49 } 50 } 51 52 return rval; 53 } 54 55 Name[] snapshotNamesFromDir(Path dir) { 56 import std.algorithm : filter, map, copy; 57 import std.array : appender; 58 import std.file : dirEntries, SpanMode, exists, isDir; 59 import std.path : baseName; 60 61 if (!dir.toString.exists) 62 return null; 63 if (!dir.toString.isDir) 64 throw new SnapshotException(SnapshotException.DstIsNotADir.init.SnapshotError); 65 66 auto app = appender!(Name[])(); 67 dirEntries(dir.toString, SpanMode.shallow).map!(a => a.name) 68 .filter!(a => a.isDir) 69 .map!(a => Name(a.baseName)) 70 .copy(app); 71 return app.data; 72 } 73 74 Name[] snapshotNamesFromSsh(const RemoteCmd cmd_, string addr, string path) { 75 import std.algorithm : map, copy; 76 import std.array : appender; 77 import std.process : execute; 78 import std..string : splitLines; 79 80 auto cmd = cmd_.match!((const SshRemoteCmd a) { 81 return a.toCmd(RemoteSubCmd.lsDirs, addr, path); 82 }); 83 if (cmd.empty) 84 return null; 85 86 auto res = execute(cmd); 87 if (res.status != 0) { 88 logger.warning(res.output); 89 return null; 90 } 91 92 auto app = appender!(Name[])(); 93 res.output.splitLines.map!(a => Name(a)).copy(app); 94 95 return app.data; 96 } 97 98 @("shall scan the directory for all snapshots") 99 unittest { 100 import std.conv : to; 101 import std.datetime : SysTime, UTC, Duration, Clock, dur; 102 import std.file; 103 import std.path; 104 import std.range : enumerate; 105 import sumtype; 106 import dsnapshot.layout : Layout, LayoutConfig, Span; 107 108 immutable tmpDir = "test_snapshot_scan"; 109 scope (exit) 110 rmdirRecurse(tmpDir); 111 mkdir(tmpDir); 112 113 auto curr = Clock.currTime; 114 curr.timezone = UTC(); 115 116 foreach (const i; 0 .. 15) { 117 mkdir(buildPath(tmpDir, curr.toISOExtString)); 118 curr -= 1.dur!"hours"; 119 } 120 foreach (const i; 0 .. 15) { 121 curr -= 5.dur!"hours"; 122 mkdir(buildPath(tmpDir, curr.toISOExtString)); 123 } 124 125 auto conf = LayoutConfig([Span(5, 4.dur!"hours"), Span(5, 1.dur!"days")]); 126 const base = Clock.currTime; 127 auto layout = Layout(base, conf); 128 layout = fillLayout(layout, FlowLocal(LocalAddr(tmpDir), LocalAddr(tmpDir)) 129 .Flow, RemoteCmd(SshRemoteCmd.init)); 130 131 layout.waiting.length.shouldEqual(0); 132 layout.discarded.length.shouldEqual(20); 133 134 (base - layout.snapshotTimeInBucket(0).get).total!"hours".shouldEqual(4); 135 (base - layout.snapshotTimeInBucket(4).get).total!"hours".shouldEqual(4 * 5); 136 (base - layout.snapshotTimeInBucket(5).get).total!"hours".shouldEqual(4 * 5 + 24 + 1); 137 (base - layout.snapshotTimeInBucket(6).get).total!"hours".shouldEqual(4 * 5 + 24 * 2 + 2); 138 (base - layout.snapshotTimeInBucket(7).get).total!"hours".shouldEqual(4 * 5 + 24 * 3 - 2); 139 140 /// these buckets are filled by the second pass 141 (base - layout.snapshotTimeInBucket(8).get).total!"hours".shouldEqual(4 * 5 + 24 * 4 - 31); 142 (base - layout.snapshotTimeInBucket(9).get).total!"hours".shouldEqual(4 * 5 + 24 * 5 - 60); 143 }