1 module unde.scan; 2 3 import unde.global_state; 4 import unde.lsblk; 5 import unde.path_mnt; 6 import unde.slash; 7 import unde.marks; 8 import unde.command_line.db; 9 10 import std.stdio; 11 import std.conv; 12 import core.stdc.stdlib; 13 import std.math; 14 import berkeleydb.all; 15 import std.stdint; 16 import core.stdc.stdlib; 17 import std..string; 18 import std.algorithm.sorting; 19 import std.algorithm.searching; 20 import std.utf; 21 import std.concurrency; 22 import core.time; 23 import core.exception; 24 import std.process; 25 import std.regex; 26 27 import derelict.sdl2.sdl; 28 import derelict.sdl2.ttf; 29 import derelict.sdl2.image; 30 31 import unde.lib; 32 33 import std.file; 34 35 version(Windows) 36 { 37 import core.sys.windows.windows; 38 import core.stdc.time; 39 alias ulong ulong_t; 40 } 41 42 immutable DRect drect_zero = DRect(0, 0, 0, 0); 43 44 private void 45 remove_deleted_files(ScannerGlobalState sgs, string mnt, string name1, string name2) 46 { 47 if (sgs.copy_map.length > 0) return; 48 Dbc cursor = sgs.db_map.cursor(sgs.txn, 0); 49 scope(exit) cursor.close(); 50 51 LsblkInfo info = sgs.lsblk[mnt]; 52 string subpath = .subpath(name1, mnt); 53 string path0 = info.uuid ~ subpath.replace(SL, "\0"); 54 55 string subpath2 = .subpath(name2, mnt); 56 string path02 = info.uuid ~ subpath2.replace(SL, "\0"); 57 58 Dbt key = path0; 59 Dbt data; 60 auto res = cursor.get(&key, &data, DB_SET_RANGE); 61 if (res != 0) return; 62 63 int removed = 0; 64 do 65 { 66 string path = key.to!string(); 67 if (path < path02) 68 { 69 DirEntry de; 70 string pathfile = path.replace("\0", SL); 71 //writefln("Remove %s", pathfile); 72 73 pathfile = mnt ~ pathfile[pathfile.indexOf(SL)..$]; 74 if (exists(pathfile)) 75 { 76 writeln(pathfile ~ " exists.\nRemoving from " ~ info.uuid ~ subpath ~ " till " ~ info.uuid ~ subpath2); 77 } 78 79 // FYI: proc system sometimes make not listable but existing 80 // directories 81 //assert (!exists(pathfile), pathfile ~ " exists.\nRemoving from " ~ info.uuid ~ subpath ~ " till " ~ info.uuid ~ subpath2); 82 removed++; 83 cursor.del(); 84 sgs.OIT++; 85 if (sgs.OIT > 100) 86 { 87 cursor.close(); 88 sgs.recommit(); 89 cursor = sgs.db_map.cursor(sgs.txn, 0); 90 res = cursor.get(&key, &data, DB_SET_RANGE); 91 if (res != 0) return; 92 } 93 } 94 else break; 95 } 96 while (cursor.get(&key, &data, DB_NEXT) == 0); 97 98 /* 99 if (removed > 0) 100 writefln("%s items removed from %s to %s", removed, info.uuid ~ subpath, info.uuid ~ subpath2); 101 */ 102 } 103 104 private void 105 copy_map_of_path(ScannerGlobalState sgs, PathMnt path, 106 PathMnt copy_to_mnt, bool move) 107 { 108 string copy_to0 = copy_to_mnt.get_key(sgs.lsblk); 109 110 begin: 111 try 112 { 113 Dbc cursor = sgs.db_map.cursor(sgs.txn, 0); 114 scope(exit) cursor.close(); 115 116 string path0 = path.get_key(sgs.lsblk); 117 Dbt key = path0; 118 Dbt data; 119 auto res = cursor.get(&key, &data, DB_SET_RANGE); 120 if (res != 0) return; 121 122 int removed = 0; 123 writefln("Start copy map %s to %s", path, copy_to_mnt); 124 do 125 { 126 receive_copy_map(sgs); 127 string path1 = key.to!string(); 128 if (path1.startsWith(path0)) 129 { 130 string path2 = path1.replace(path0, copy_to0); 131 Dbt key2 = path2; 132 //writefln("Write %s", path2.replace("\0", SL)); 133 res = sgs.db_map.put(sgs.txn, &key2, &data); 134 if (res != 0) 135 throw new Exception("Path info to map-db not written"); 136 sgs.OIT++; 137 if (sgs.OIT > 100) 138 { 139 cursor.close(); 140 sgs.recommit(); 141 cursor = sgs.db_map.cursor(sgs.txn, 0); 142 res = cursor.get(&key, &data, DB_SET_RANGE); 143 if (res != 0) return; 144 } 145 } 146 else break; 147 if (move) 148 { 149 Dbc cursor2; 150 cursor2 = sgs.db_command_output.cursor(sgs.txn, 0); 151 scope(exit) cursor2.close(); 152 153 Dbt key2, data2; 154 string ks = get_key_for_command_out(command_out_key(path1, 0, 0)); 155 key2 = ks; 156 res = cursor2.get(&key2, &data2, DB_SET_RANGE); 157 if (res == 0) 158 { 159 do 160 { 161 string key_string = key2.to!(string); 162 command_out_key cmd_out_key; 163 parse_key_for_command_out(key_string, cmd_out_key); 164 165 if (cmd_out_key.cwd == path1) 166 { 167 sgs.OIT++; 168 if (sgs.is_time_to_recommit()) 169 { 170 cursor.close(); 171 sgs.recommit(); 172 cursor = sgs.db_map.cursor(sgs.txn, 0); 173 res = cursor.get(&key, &data, DB_SET_RANGE); 174 if (res != 0) return; 175 176 cursor2.close(); 177 sgs.recommit(); 178 cursor2 = sgs.db_command_output.cursor(sgs.txn, 0); 179 res = cursor2.get(&key2, &data2, DB_SET_RANGE); 180 if (res == DB_NOTFOUND) 181 { 182 return; 183 } 184 } 185 186 string path2 = path1.replace(path0, copy_to0); 187 ks = get_key_for_command_out(command_out_key(path2, 188 cmd_out_key.cmd_id, cmd_out_key.out_id)); 189 Dbt key3 = ks; 190 191 res = sgs.db_command_output.put(sgs.txn, &key3, &data2); 192 if (res != 0) 193 throw new Exception("Info to command out-db not written"); 194 195 cursor2.del(); 196 } 197 else 198 { 199 break; 200 } 201 202 } while (cursor2.get(&key2, &data2, DB_NEXT) == 0); 203 } 204 205 Dbc cursor3; 206 cursor3 = sgs.db_commands.cursor(sgs.txn, 0); 207 scope(exit) cursor3.close(); 208 209 Dbt key3, data3; 210 string ks3 = get_key_for_command(command_key(path1, 0)); 211 key3 = ks3; 212 res = cursor3.get(&key3, &data3, DB_SET_RANGE); 213 if (res == 0) 214 { 215 do 216 { 217 string key_string = key3.to!(string); 218 command_key cmd_key; 219 parse_key_for_command(key_string, cmd_key); 220 221 if (cmd_key.cwd == path1) 222 { 223 sgs.OIT++; 224 if (sgs.is_time_to_recommit()) 225 { 226 cursor.close(); 227 sgs.recommit(); 228 cursor = sgs.db_map.cursor(sgs.txn, 0); 229 res = cursor.get(&key, &data, DB_SET_RANGE); 230 if (res != 0) return; 231 232 cursor3.close(); 233 sgs.recommit(); 234 cursor3 = sgs.db_commands.cursor(sgs.txn, 0); 235 res = cursor3.get(&key3, &data3, DB_SET_RANGE); 236 if (res == DB_NOTFOUND) 237 { 238 return; 239 } 240 } 241 242 string path2 = path1.replace(path0, copy_to0); 243 ks3 = get_key_for_command(command_key(path2, 244 cmd_key.id)); 245 Dbt key4 = ks3; 246 247 res = sgs.db_commands.put(sgs.txn, &key4, &data3); 248 if (res != 0) 249 throw new Exception("Info to command out-db not written"); 250 251 cursor3.del(); 252 } 253 else 254 { 255 break; 256 } 257 258 } while (cursor3.get(&key3, &data3, DB_NEXT) == 0); 259 } 260 261 cursor.del(); 262 sgs.OIT++; 263 } 264 } 265 while (cursor.get(&key, &data, DB_NEXT) == 0); 266 267 if (move) 268 { 269 foreach (char c; "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789") 270 { 271 string m = "" ~ c; 272 key = m; 273 res = sgs.db_marks.get(null, &key, &data); 274 if (res == 0) 275 { 276 Mark mark = data.to!(Mark); 277 string mpath = from_char_array(mark.path); 278 if (mpath.startsWith(path0)) 279 { 280 mpath = mpath.replace(path0, copy_to0); 281 mark.path = to_char_array!MARKS_PATH_MAX(mpath); 282 data = mark; 283 res = sgs.db_marks.put(null, &key, &data); 284 if (res != 0) 285 throw new Exception("Mark info to marks-db not written"); 286 } 287 } 288 } 289 } 290 } 291 catch (DbDeadlockException exp) 292 { 293 writefln("Oops deadlock, retry"); 294 sgs.abort(); 295 sgs.recommit(); 296 goto begin; 297 } 298 299 writefln("End copy map"); 300 } 301 302 private void 303 fixsizes_for_parents(ScannerGlobalState sgs, PathMnt path, 304 long dsize, long ddisk_usage, long dfiles) 305 { 306 string path0 = path.get_key(sgs.lsblk); 307 path0 = path0[0..path0.lastIndexOf("\0")]; 308 while(path0.lastIndexOf("\0") >= 0) 309 { 310 Dbt key = path0; 311 Dbt data; 312 auto res = sgs.db_map.get(sgs.txn, &key, &data); 313 if (res == 0) 314 { 315 RectSize rectsize = data.to!(RectSize); 316 317 //writef("%s diskusage = %s + %s", path0.replace("\0",SL), rectsize.disk_usage, ddisk_usage); 318 rectsize.size += dsize; 319 rectsize.disk_usage += ddisk_usage; 320 rectsize.files += dfiles; 321 //writefln("= %s", rectsize.disk_usage); 322 323 data = rectsize; 324 res = sgs.db_map.put(sgs.txn, &key, &data); 325 if (res != 0) 326 throw new Exception("Path info to map-db not written"); 327 sgs.OIT++; 328 sgs.recommit(); 329 } 330 331 if (path0.lastIndexOf("\0") == path0.length-1) 332 { 333 path0 = ""; 334 } 335 else 336 { 337 path0 = path0[0..path0.lastIndexOf("\0")]; 338 if (path0.lastIndexOf("\0") < 0) 339 { 340 path0 ~= "\0"; 341 } 342 } 343 } 344 } 345 346 private void 347 receive_copy_map(ScannerGlobalState sgs) 348 { 349 receiveTimeout( 0.seconds, 350 (string path, string copy_to_mnt, bool move) 351 { 352 auto copymapinfo = CopyMapInfo(path, null, move, thisTid); 353 sgs.copy_map[copy_to_mnt] = copymapinfo; 354 writefln("scan.thread send(%s, %s, %s, %s)", 355 path, copy_to_mnt, move, MsgState.Received); 356 send(sgs.parent_tid, thisTid, path, copy_to_mnt, move, MsgState.Received); 357 } 358 ); 359 } 360 361 private RectSize 362 scan_direntry(ScannerGlobalState sgs, PathMnt path, 363 DRect rect, bool recursively, ref bool cont, int rescan, bool root) 364 { 365 sgs.recommit(); 366 367 //writefln("%s", path); 368 receive_copy_map(sgs); 369 370 bool orig_cont = cont; 371 bool fully_scanned = false; 372 string path0 = path.get_key(sgs.lsblk); 373 Dbt key = path0; 374 Dbt data; 375 RectSize rectsize; 376 RectSize rectsize_before; 377 378 if (cont || rescan) 379 { 380 auto res = sgs.db_map.get(sgs.txn, &key, &data); 381 if (res == 0) 382 { 383 rectsize = data.to!(RectSize); 384 385 if (rectsize.size >= 0) 386 { 387 fully_scanned = true; 388 rectsize = rectsize; 389 } 390 } 391 if (sgs.one_level) fully_scanned = true; 392 if (cont && !fully_scanned) writefln("Continue from %s", path); 393 if (!fully_scanned) cont = false; 394 } 395 rectsize_before = rectsize; 396 397 //writefln("path=%s in copy_map=%s", path, sgs.copy_map); 398 if (path in sgs.copy_map) 399 { 400 //writefln("scan.thread Found path=%s", path); 401 PathMnt copy_to_mnt = PathMnt(sgs.lsblk, path); 402 PathMnt path_from_mnt = PathMnt(sgs.lsblk, sgs.copy_map[path].path); 403 404 string path_from0 = path_from_mnt.get_key(sgs.lsblk); 405 Dbt key2 = path_from0; 406 Dbt data2; 407 RectSize rectsize2; 408 409 auto res = sgs.db_map.get(sgs.txn, &key2, &data2); 410 //writefln("scan.thread GET %s", path_from0.replace("\0", SL)); 411 if (res == 0) 412 { 413 //writefln("GOT"); 414 rectsize2 = data2.to!(RectSize); 415 416 rectsize.size = rectsize2.size; 417 rectsize.disk_usage = rectsize2.disk_usage; 418 rectsize.mtime = rectsize2.mtime; 419 rectsize.mtime_nsec = rectsize2.mtime_nsec; 420 rectsize.files = rectsize2.files; 421 } 422 423 if (rect.x > 0) 424 rectsize.rect_by_name = rect; 425 426 data = rectsize; 427 res = sgs.db_map.put(sgs.txn, &key, &data); 428 if (res != 0) 429 throw new Exception("Path info to map-db not written"); 430 sgs.OIT++; 431 432 path_from_mnt._next = path_from_mnt._next ~ SL; 433 copy_to_mnt._next = copy_to_mnt._next ~ SL; 434 435 copy_map_of_path(sgs, path_from_mnt, copy_to_mnt, sgs.copy_map[path].move); 436 /*writefln("scan.thread send to %s - %s, %s, %s, used", 437 sgs.parent_tid, thisTid, sgs.copy_map[path].path, path);*/ 438 send(sgs.parent_tid, thisTid, sgs.copy_map[path].path, path._next, sgs.copy_map[path].move, MsgState.Used); 439 sgs.copy_map.remove(path); 440 441 if (root) 442 { 443 long dsize = rectsize.size - rectsize_before.size; 444 long ddisk_usage = rectsize.disk_usage - rectsize_before.disk_usage; 445 long dfiles = rectsize.files - rectsize_before.files; 446 447 //writefln("%s - size=%s, disk_usage=%s, dfiles=%s", 448 // path, dsize, ddisk_usage, dfiles); 449 450 fixsizes_for_parents(sgs, path, 451 dsize, ddisk_usage, dfiles); 452 } 453 454 return rectsize; 455 } 456 457 //writefln("%s: %s", path, fully_scanned); 458 459 DRect rescan_rect; 460 if (rescan > 0 ? rescan == 1 || rescan == 2 && !fully_scanned : !cont) 461 { 462 if (rescan != 1) 463 rectsize = RectSize(rect, drect_zero, drect_zero, -1); 464 data = rectsize; 465 auto res = sgs.db_map.put(sgs.txn, &key, &data); 466 if (res != 0) 467 throw new Exception("Path info to map-db not written"); 468 sgs.OIT++; 469 470 rescan_rect = rectsize.rect_by_name; 471 rectsize = walk(sgs, path, recursively, orig_cont, 472 rescan == 1 ? 2 : 0); 473 } 474 475 if (path.path == path._next && !(rescan || !cont)) 476 { 477 rectsize.rect_by_size = rect; 478 rectsize.rect_by_time = rect; 479 rescan = 2; 480 } 481 482 if (rescan || !cont) 483 { 484 if (rescan == 1) 485 rectsize.rect_by_name = rescan_rect; 486 else 487 rectsize.rect_by_name = rect; 488 data = rectsize; 489 auto res = sgs.db_map.put(sgs.txn, &key, &data); 490 if (res != 0) 491 throw new Exception("Path info to map-db not written"); 492 sgs.OIT++; 493 } 494 495 if (root) 496 { 497 long dsize = rectsize.size - rectsize_before.size; 498 long ddisk_usage = rectsize.disk_usage - rectsize_before.disk_usage; 499 long dfiles = rectsize.files - rectsize_before.files; 500 501 //writefln("%s - size=%s, disk_usage=%s, dfiles=%s", 502 // path, dsize, ddisk_usage, dfiles); 503 504 fixsizes_for_parents(sgs, path, 505 dsize, ddisk_usage, dfiles); 506 } 507 508 return rectsize; 509 } 510 511 public void 512 calculate_rect(DRect full_rect, out DRect rect, const(long)[] coords, size_t lev) 513 { 514 if (lev == 0) 515 { 516 rect.w = full_rect.w/3; 517 rect.h = full_rect.h/3; 518 rect.x = full_rect.x + rect.w; 519 rect.y = full_rect.y + rect.h; 520 } 521 else 522 { 523 rect.w = full_rect.w/3/exp2(lev); 524 rect.h = full_rect.h/3/exp2(lev); 525 526 rect.x = full_rect.x + full_rect.w/3; 527 rect.y = full_rect.y + full_rect.w/3/exp2(lev); 528 529 /* right, down, left, up, right2 */ 530 long r = 2, dlu = 3, r2 = 1; 531 foreach (j; 1..lev) 532 { 533 r = r*2+2; 534 dlu = dlu*2+3; 535 r2 = r2*2+1; 536 } 537 538 // right 539 if (coords[lev] < r) 540 { 541 rect.x += full_rect.w/3/exp2(lev) * coords[lev]; 542 } 543 else 544 { 545 rect.x += full_rect.w/3/exp2(lev) * r; 546 547 // down 548 if (coords[lev] < r+dlu) 549 { 550 rect.y += full_rect.w/3/exp2(lev) * (coords[lev]-r); 551 } 552 else 553 { 554 rect.y += full_rect.w/3/exp2(lev) * dlu; 555 556 // left 557 if (coords[lev] < r+2*dlu) 558 { 559 rect.x -= full_rect.w/3/exp2(lev) * (coords[lev]-r-dlu); 560 } 561 else 562 { 563 rect.x -= full_rect.w/3/exp2(lev) * dlu; 564 565 // up 566 if (coords[lev] < r+3*dlu) 567 { 568 rect.y -= full_rect.w/3/exp2(lev) * (coords[lev]-r-2*dlu); 569 } 570 else 571 { 572 rect.y -= full_rect.w/3/exp2(lev) * dlu; 573 574 // right 575 if (coords[lev] < r+3*dlu+r2) 576 { 577 rect.x += full_rect.w/3/exp2(lev) * (coords[lev]-r-3*dlu); 578 } 579 else 580 { 581 rect.x += full_rect.w/3/exp2(lev) * dlu; 582 } 583 } 584 } 585 } 586 } 587 } 588 } 589 590 public void 591 calculate_coords(long[] coords, ref size_t lev, ref long i, 592 int levels, immutable long entries_on_last_level) 593 { 594 coords[lev]++; 595 i++; 596 lev++; 597 /*foreach(j; 1..lev) 598 { 599 writef("%s, ", coords[j]); 600 } 601 writefln(""); 602 writefln("coords[levels]=%s, entries_on_last_level=%s", coords[levels], entries_on_last_level); 603 */ 604 if (lev > levels || coords[levels] >= entries_on_last_level && lev >= levels) 605 { 606 bool repeat = false; 607 do 608 { 609 lev--; 610 if (lev == 1) break; 611 612 /* right, down, left, up */ 613 long r = 4, rd = 5, d = 4, r1 = 0; 614 foreach (j; 2..lev) 615 { 616 r = r*2+4; 617 d = d*2+8; 618 r1 = r1*2+4; 619 } 620 //writefln("lev=%s, r=%s, d=%s, r1=%s", lev, r, d, r1); 621 622 long c = coords[lev]; 623 624 // right 625 if (c <= r) 626 { 627 repeat = ((c%2)==0); 628 } 629 else 630 { 631 // right, down 632 c -= r; 633 if (c <= rd) 634 { 635 repeat = (c == 5); 636 } 637 else 638 { 639 // down 640 c -= rd; 641 if (c <= d) 642 { 643 repeat = ((c%2)==0); 644 } 645 else 646 { 647 // down, left 648 c -= d; 649 if (c <= rd) 650 { 651 repeat = (c == 5); 652 } 653 else 654 { 655 // left 656 c -= rd; 657 if (c <= d) 658 { 659 repeat = ((c%2)==0); 660 } 661 else 662 { 663 // left, up 664 c -= d; 665 if (c <= rd) 666 { 667 repeat = (c == 5); 668 } 669 else 670 { 671 // up 672 c -= rd; 673 if (c <= d) 674 { 675 repeat = ((c%2)==0); 676 } 677 else 678 { 679 // up, right 680 c -= d; 681 if (c <= rd) 682 { 683 repeat = (c == 5); 684 } 685 else 686 { 687 // right 688 c -= rd; 689 if (c <= r1) 690 { 691 repeat = ((c%2)==0); 692 } 693 else 694 { 695 assert(false); 696 } 697 } 698 } 699 } 700 } 701 } 702 } 703 } 704 } 705 } 706 while(repeat); 707 708 if (coords[levels] >= entries_on_last_level && lev >= levels) 709 lev--; 710 } 711 } 712 713 private RectSize 714 rectsize_from_exception(Exception e) 715 { 716 auto rectsize = RectSize(); 717 rectsize.msg_color = 0x80808000; //ARGB 718 rectsize.msg = to_char_array!80(strip_error(e.msg)); 719 return rectsize; 720 } 721 722 version (Windows) 723 { 724 ulong getFileSizeOnDisk(string path) 725 { 726 DWORD high_size; 727 auto size = GetCompressedFileSize(path.toUTF16z(), &high_size); 728 if (size == INVALID_FILE_SIZE) 729 { 730 throw new Exception("GetCompressedFileSize: "~GetErrorMessage()); 731 } 732 733 return (cast(ulong)high_size << 32) | size; 734 } 735 736 ulong getFileSize(string path) 737 { 738 HANDLE file = CreateFile( 739 path.toUTF16z(), 740 GENERIC_READ, 741 7, 742 null, 743 OPEN_EXISTING, 744 FILE_FLAG_BACKUP_SEMANTICS, 745 null 746 ); 747 if (file == INVALID_HANDLE_VALUE) 748 { 749 writefln("CreateFile %s: %s", path, GetErrorMessage()); 750 return 0; 751 } 752 753 DWORD high_size; 754 auto size = GetFileSize(file, &high_size); 755 if (size == INVALID_FILE_SIZE) 756 { 757 CloseHandle(file); 758 throw new Exception("GetFileSize: "~GetErrorMessage()); 759 } 760 761 auto res = CloseHandle(file); 762 if (!res) 763 { 764 throw new Exception("CloseHandle: "~GetErrorMessage()); 765 } 766 767 return (cast(ulong)high_size << 32) | size; 768 } 769 770 time_t[2] getFileModTime(string path) 771 { 772 HANDLE file = CreateFile( 773 path.toUTF16z(), 774 GENERIC_READ, 775 7, 776 null, 777 OPEN_EXISTING, 778 FILE_FLAG_BACKUP_SEMANTICS, 779 null 780 ); 781 if (file == INVALID_HANDLE_VALUE) 782 { 783 writefln("CreateFile %s: %s", path, GetErrorMessage()); 784 return [0, 0]; 785 } 786 787 FILETIME filetime; 788 auto res = GetFileTime(file, null, null, &filetime); 789 if (!res) 790 { 791 CloseHandle(file); 792 throw new Exception("GetFileTime: "~GetErrorMessage()); 793 } 794 795 res = CloseHandle(file); 796 if (!res) 797 { 798 throw new Exception("CloseHandle: "~GetErrorMessage()); 799 } 800 801 ulong windows_time = (cast(ulong)filetime.dwHighDateTime << 32) | filetime.dwLowDateTime; 802 803 enum WINDOWS_TICK=10000000; 804 enum SEC_TO_UNIX_EPOCH=11644473600; 805 806 return [ cast(time_t)(windows_time / WINDOWS_TICK - SEC_TO_UNIX_EPOCH), 807 cast(time_t)(windows_time % WINDOWS_TICK) ]; 808 } 809 810 } 811 812 private RectSize 813 walk(ScannerGlobalState sgs, PathMnt path, 814 bool recursively, bool cont, int rescan) 815 { 816 sgs.recommit(); 817 DRect full_rect = DRect(0, 0, 1024*1024, 1024*1024); 818 // receive any messages and exits on exits of parent 819 receiveTimeout( 0.seconds, 820 (OwnerTerminated ot) { 821 writefln("Abort scanning due stopping parent"); 822 sgs.finish = true; 823 } ); 824 825 if (sgs.finish) 826 return RectSize(drect_zero, drect_zero, drect_zero, -1); 827 828 DirEntry de; 829 try 830 { 831 de = DirEntry(path); 832 } 833 catch (FileException e) 834 { 835 return rectsize_from_exception(e); 836 } 837 838 if (path in sgs.lsblk) 839 { 840 //writefln("path=%s, recursively=%s, root=%s", path, 841 // recursively, path.path == path); 842 if (!recursively && path.path != path) 843 { 844 //writefln("EXIT!"); 845 return RectSize(); 846 } 847 } 848 849 bool np = (path._next != path.path); 850 851 try 852 { 853 path.update(sgs.lsblk); 854 } 855 catch (Exception e) 856 { 857 return rectsize_from_exception(e); 858 } 859 860 if (path in sgs.lsblk && np) 861 { 862 DRect rect = DRect(0, 0, 1024*1024, 1024*1024); 863 scan_direntry(sgs, path, rect, recursively, cont, rescan, false); 864 version(Posix) 865 { 866 return RectSize(drect_zero, drect_zero, drect_zero, 867 0, de.statBuf.st_blocks*512, 868 de.statBuf.st_mtime, de.statBuf.st_mtimensec, 0); 869 } 870 else version(Windows) 871 { 872 time_t[2] modtime = getFileModTime(path); 873 return RectSize(drect_zero, drect_zero, drect_zero, 874 getFileSize(path), getFileSizeOnDisk(path), 875 modtime[0], modtime[1], 0); 876 } 877 } 878 879 bool isSymlink = false; 880 try 881 { 882 isSymlink = de.isSymlink; 883 } 884 catch (Exception e) 885 { 886 return rectsize_from_exception(e); 887 } 888 889 if (isSymlink) 890 { 891 return RectSize(drect_zero, drect_zero, drect_zero, 0, 0, 0, 0, 1); 892 } 893 else if (de.isDir) 894 { 895 string[] paths; 896 897 try 898 { 899 foreach (string name; dirEntries(path, SpanMode.shallow)) 900 { 901 paths ~= name; 902 } 903 } 904 catch (FileException e) 905 { 906 return rectsize_from_exception(e); 907 } 908 909 sort!("a < b")(paths); 910 911 int levels = 0; 912 long l = paths.length; 913 /*if (l > 0) 914 { 915 l--; 916 }*/ 917 for (long i=12; i < l; i=i*2+12) 918 { 919 l -= i; 920 levels++; 921 } 922 if (l > 0) levels++; 923 924 immutable long entries_on_last_level = l; 925 926 RectSize[string] rect_sizes; 927 928 //writefln("levels = %s", levels); 929 long[] coords = new long[levels+1]; 930 version(Posix) 931 { 932 RectSize full_size = RectSize(drect_zero, drect_zero, drect_zero, 933 0, de.statBuf.st_blocks*512, 934 de.statBuf.st_mtime, de.statBuf.st_mtimensec, 0); 935 } 936 else version(Windows) 937 { 938 time_t[2] modtime = getFileModTime(path); 939 RectSize full_size = RectSize(drect_zero, drect_zero, drect_zero, 940 getFileSize(path), getFileSizeOnDisk(path), 941 modtime[0], modtime[1], 0); 942 } 943 long i = 0; 944 size_t lev = 1; 945 string prev_name = path; 946 bool first = true; 947 foreach (string name; paths) 948 { 949 if (name != path) 950 { 951 DRect rect; 952 calculate_rect(full_rect, rect, coords, lev); 953 954 remove_deleted_files(sgs, path.mnt, (first ? 955 prev_name~"/\x00" : 956 prev_name~"/\xFF"), name); 957 prev_name = name; 958 RectSize rect_size = scan_direntry(sgs, path.next(name), 959 rect, recursively, cont, rescan, false); 960 if (rect_size.size < 0) return RectSize(drect_zero, drect_zero, drect_zero, -1); 961 full_size.size += rect_size.size; 962 full_size.disk_usage += rect_size.disk_usage; 963 full_size.files += rect_size.files; 964 965 rect_sizes[name] = rect_size; 966 967 calculate_coords(coords, lev, i, levels, entries_on_last_level); 968 first = false; 969 } 970 } 971 972 remove_deleted_files(sgs, path.mnt, prev_name~"/\xFF", path~"/\xFF"); 973 974 sort!((a, b) => rect_sizes[a].disk_usage > rect_sizes[b].disk_usage || 975 rect_sizes[a].disk_usage == rect_sizes[b].disk_usage && a < b)(paths); 976 coords = new long[levels+1]; 977 i = 0, lev = 1; 978 foreach (string name; paths) 979 { 980 if (name != path) 981 { 982 DRect rect; 983 calculate_rect(full_rect, rect, coords, lev); 984 rect_sizes[name].rect_by_size = rect; 985 calculate_coords(coords, lev, i, levels, entries_on_last_level); 986 } 987 } 988 989 sort!((a, b) => rect_sizes[a].mtime > rect_sizes[b].mtime || 990 rect_sizes[a].mtime == rect_sizes[b].mtime && a < b)(paths); 991 coords = new long[levels+1]; 992 i = 0, lev = 1; 993 foreach (string name; paths) 994 { 995 if (name != path) 996 { 997 DRect rect; 998 calculate_rect(full_rect, rect, coords, lev); 999 rect_sizes[name].rect_by_time = rect; 1000 1001 path = name; 1002 string path0 = path.get_key(sgs.lsblk); 1003 Dbt key = path0; 1004 Dbt data = rect_sizes[name]; 1005 assert(rect_sizes[name].rect_by_time.w > 0); 1006 //writefln("WRITE %s - %s", name, rect_sizes[name]); 1007 auto res = sgs.db_map.put(sgs.txn, &key, &data); 1008 if (res != 0) 1009 throw new Exception("Path info to map-db not written"); 1010 sgs.OIT++; 1011 1012 calculate_coords(coords, lev, i, levels, entries_on_last_level); 1013 } 1014 } 1015 1016 1017 //writefln("E %s %sx%s, size %d, density %.5f bytes/pix^2", full_rect.path, full_rect.w, full_rect.h, full_rect.size, cast(double)(full_rect.size)/(full_rect.w*full_rect.h)); 1018 return full_size; 1019 } 1020 else if (de.isFile) 1021 { 1022 try{ 1023 version(Posix) 1024 { 1025 return RectSize(drect_zero, drect_zero, drect_zero, 1026 de.size, de.statBuf.st_blocks*512, 1027 de.statBuf.st_mtime, de.statBuf.st_mtimensec, 1); 1028 } 1029 else version(Windows) 1030 { 1031 time_t[2] modtime = getFileModTime(path); 1032 return RectSize(drect_zero, drect_zero, drect_zero, 1033 getFileSize(path), getFileSizeOnDisk(path), 1034 modtime[0], modtime[1], 1); 1035 } 1036 } 1037 catch (Exception e) 1038 { 1039 return RectSize(drect_zero, drect_zero, drect_zero, -1); 1040 } 1041 } 1042 else 1043 { 1044 try 1045 { 1046 version(Posix) 1047 { 1048 return RectSize(drect_zero, drect_zero, drect_zero, 1049 de.size, de.statBuf.st_blocks*512, 1050 de.statBuf.st_mtime, de.statBuf.st_mtimensec, 1); 1051 } 1052 else version(Windows) 1053 { 1054 time_t[2] modtime = getFileModTime(path); 1055 return RectSize(drect_zero, drect_zero, drect_zero, 1056 getFileSize(path), getFileSizeOnDisk(path), 1057 modtime[0], modtime[1], 1); 1058 } 1059 } 1060 catch (Exception e) 1061 { 1062 return RectSize(drect_zero, drect_zero, drect_zero, -1); 1063 } 1064 } 1065 } 1066 1067 private void 1068 scan(shared LsblkInfo[string] lsblk, shared CopyMapInfo[string] copy_map, 1069 PathMnt path, bool recursively, bool cont, bool rescan, 1070 bool one_level, Tid tid) 1071 { 1072 ScannerGlobalState sgs = new ScannerGlobalState(); 1073 try { 1074 scope(exit) 1075 { 1076 destroy(sgs); 1077 } 1078 sgs.one_level = one_level; 1079 sgs.lsblk = to!(LsblkInfo[string])(lsblk); 1080 sgs.copy_map = cast(CopyMapInfo[string])(copy_map); 1081 sgs.parent_tid = tid; 1082 1083 DRect rect = DRect(0, 0, 1024*1024, 1024*1024); 1084 scan_direntry(sgs, path, rect, recursively, cont, rescan?1:0, true); 1085 sgs.commit(); 1086 receive_copy_map(sgs); 1087 } catch (DbDeadlockException exc) { 1088 writefln("Oops! Seems Deadlock.."); 1089 writefln("Scanning %s interrupted unexpectedly", path); 1090 writefln("Will be continued immediatedly.."); 1091 send(tid, thisTid, path); 1092 return; 1093 } catch (shared(Throwable) exc) { 1094 send(tid, exc); 1095 } 1096 1097 writefln("Finish scan %s", path); 1098 send(tid, thisTid); 1099 } 1100 1101 public int 1102 start_scan(GlobalState gs, PathMnt path) 1103 { 1104 shared LsblkInfo[string] lsblk = to!(shared LsblkInfo[string])(gs.lsblk); 1105 shared CopyMapInfo[string] copy_map = cast(shared CopyMapInfo[string])(gs.copy_map); 1106 1107 writefln("Start scan %s", path); 1108 auto tid = spawn(&scan, lsblk, copy_map, path, gs.interface_flags[path], false, false, false, thisTid); 1109 gs.scanners ~= tid; 1110 return 0; 1111 } 1112 1113 public int 1114 rescan_path(GlobalState gs, PathMnt path) 1115 { 1116 shared LsblkInfo[string] lsblk = to!(shared LsblkInfo[string])(gs.lsblk); 1117 shared CopyMapInfo[string] copy_map = cast(shared CopyMapInfo[string])(gs.copy_map.dup); 1118 1119 if (gs.rescanners.length > 0) return 0; 1120 if (gs.scanners.length > 0) return 0; 1121 if (path in gs.rescanners) return 0; 1122 1123 writefln("Rescan path %s", path); 1124 auto tid = spawn(&scan, lsblk, copy_map, path, false, false, true, gs.copiers.length > 0, thisTid); 1125 gs.scanners ~= tid; 1126 gs.rescanners[path] = tid; 1127 return 0; 1128 } 1129 1130 public void 1131 check_if_fully_scanned(GlobalState gs, string path) 1132 { 1133 static bool[string] checked; 1134 LsblkInfo info = gs.lsblk[path]; 1135 if (info.uuid in checked) 1136 { 1137 return; 1138 } 1139 1140 string k = info.uuid ~ "\0"; 1141 Dbt key = k; 1142 Dbt data; 1143 auto res = gs.db_map.get(null, &key, &data); 1144 if (res == 0) 1145 { 1146 RectSize rectsize; 1147 rectsize = data.to!(RectSize); 1148 1149 if (rectsize.rect_by_time.w > 0) 1150 { 1151 writefln("Fully scanned %s FS detected (%s)", info.uuid, path); 1152 } 1153 else 1154 { 1155 writefln("Not fully scanned %s FS detected (%s)", info.uuid, path); 1156 writefln("Continue scanning"); 1157 shared LsblkInfo[string] lsblk = to!(shared LsblkInfo[string])(gs.lsblk); 1158 shared CopyMapInfo[string] copy_map = cast(shared CopyMapInfo[string])(gs.copy_map.dup); 1159 auto tid = spawn(&scan, lsblk, copy_map, PathMnt(gs.lsblk, path), false, true, false, false, thisTid); 1160 gs.scanners ~= tid; 1161 } 1162 } 1163 else 1164 { 1165 writefln("Not started to scan %s FS detected (%s)", info.uuid, path); 1166 } 1167 1168 checked[info.uuid] = true; 1169 } 1170 1171 enum MsgState 1172 { 1173 Send, 1174 Resent, 1175 Received, 1176 Used 1177 } 1178 1179 public void 1180 check_scanners(GlobalState gs) 1181 { 1182 if (gs.scanners.length > 0 || gs.removers.length > 0 || 1183 gs.copiers.length > 0 || gs.movers.length > 0 || 1184 gs.changers_rights.length > 0 || gs.commands.length > 0 || 1185 gs.delete_commands.length > 0) 1186 { 1187 gs.dirty = true; 1188 while(receiveTimeout( 0.seconds, 1189 (Tid tid, PathMnt path) 1190 { 1191 bool is_scanner; 1192 foreach(i, scanner; gs.scanners) 1193 { 1194 if (scanner == tid) 1195 { 1196 is_scanner = true; 1197 gs.scanners = gs.scanners[0..i]~gs.scanners[i+1..$]; 1198 } 1199 } 1200 1201 bool is_rescanner; 1202 foreach(path, t; gs.rescanners) 1203 { 1204 if (t == tid) 1205 { 1206 is_rescanner = true; 1207 gs.rescanners.remove(path); 1208 break; 1209 } 1210 } 1211 1212 if (is_scanner) 1213 { 1214 shared LsblkInfo[string] lsblk = to!(shared LsblkInfo[string])(gs.lsblk); 1215 shared CopyMapInfo[string] copy_map = cast(shared CopyMapInfo[string])(gs.copy_map.dup); 1216 auto rescanner_tid = spawn(&scan, lsblk, copy_map, path, false, true, false, false, thisTid); 1217 gs.scanners ~= rescanner_tid; 1218 } 1219 else if (is_rescanner) 1220 { 1221 rescan_path(gs, path); 1222 } 1223 }, 1224 (Tid tid) 1225 { 1226 bool is_scanner; 1227 foreach(i, scanner; gs.scanners) 1228 { 1229 if (scanner == tid) 1230 { 1231 is_scanner = true; 1232 gs.scanners = gs.scanners[0..i]~gs.scanners[i+1..$]; 1233 } 1234 } 1235 1236 foreach(path, t; gs.rescanners) 1237 { 1238 if (t == tid) 1239 { 1240 is_scanner = true; 1241 gs.rescanners.remove(path); 1242 break; 1243 } 1244 } 1245 1246 if (is_scanner) 1247 { 1248 foreach (copy_map_info; gs.copy_map) 1249 { 1250 if (copy_map_info.sent.remove(tid)) 1251 { 1252 //writefln("main.thread sent-- = %d", copy_map_info.sent.length); 1253 1254 if (copy_map_info.sent.length == 0) 1255 { 1256 //writefln("main.thread send(resent) to %s (after got finish from %s)", copy_map_info.from, tid); 1257 send(copy_map_info.from, MsgState.Resent); 1258 } 1259 } 1260 } 1261 } 1262 else if (tid in gs.removers) 1263 { 1264 string[] paths = gs.removers[tid]; 1265 bool[string] parents; 1266 foreach (path; paths) 1267 { 1268 if (!exists(path)) 1269 { 1270 gs.selection_hash.remove(path); 1271 } 1272 string parent = getParent(path); 1273 parents[parent] = true; 1274 } 1275 1276 foreach(parent, t; parents) 1277 { 1278 rescan_path(gs, PathMnt(gs.lsblk, parent)); 1279 } 1280 1281 calculate_selection_sub(gs); 1282 gs.removers.remove(tid); 1283 } 1284 else if (tid in gs.copiers) 1285 { 1286 string[] paths = gs.copiers[tid]; 1287 bool[string] parents; 1288 foreach (path; paths) 1289 { 1290 if (!gs.selection_hash.remove(path)) 1291 { 1292 path = path[0..$-1]; 1293 if (!gs.selection_hash.remove(path)) 1294 { 1295 writefln("Can't unselect %s", path); 1296 } 1297 } 1298 string parent = getParent(path); 1299 if (parent == "") parent = SL; 1300 parents[parent] = true; 1301 } 1302 1303 calculate_selection_sub(gs); 1304 gs.copiers.remove(tid); 1305 1306 foreach(parent, t; parents) 1307 { 1308 rescan_path(gs, PathMnt(gs.lsblk, parent)); 1309 } 1310 } 1311 else if (tid in gs.movers) 1312 { 1313 string[] paths = gs.movers[tid]; 1314 bool[string] parents; 1315 foreach (path; paths) 1316 { 1317 if (!gs.selection_hash.remove(path)) 1318 { 1319 path = path[0..$-1]; 1320 if (!exists(path)) 1321 { 1322 if (!gs.selection_hash.remove(path)) 1323 { 1324 writefln("Can't unselect %s", path); 1325 } 1326 } 1327 } 1328 string parent = getParent(path); 1329 if (parent == "") parent = SL; 1330 parents[parent] = true; 1331 } 1332 1333 calculate_selection_sub(gs); 1334 gs.movers.remove(tid); 1335 1336 foreach(parent, t; parents) 1337 { 1338 rescan_path(gs, PathMnt(gs.lsblk, parent)); 1339 } 1340 } 1341 else if (tid in gs.changers_rights) 1342 { 1343 gs.changers_rights.remove(tid); 1344 } 1345 else if (tid in gs.commands) 1346 { 1347 if (gs.command_line.command_in_focus_tid == tid) 1348 { 1349 gs.command_line.command_in_focus_id = 0; 1350 if (gs.command_line.terminal) 1351 { 1352 gs.command_line.enter = true; 1353 gs.keybar.input_mode = true; 1354 } 1355 } 1356 1357 gs.commands.remove(tid); 1358 foreach(id, t; gs.tid_by_command_id) 1359 { 1360 if (t == tid) 1361 { 1362 gs.tid_by_command_id.remove(id); 1363 break; 1364 } 1365 } 1366 } 1367 else if (tid in gs.delete_commands) 1368 { 1369 gs.delete_commands.remove(tid); 1370 } 1371 else 1372 { 1373 throw new Exception("UNKNOWN TID"); 1374 } 1375 }, 1376 (Tid tid, string msg, ulong new_id) 1377 { 1378 //writefln("new_id=%s, tid=%s", new_id, tid); 1379 if (msg != "command_id") 1380 { 1381 throw new Exception("Unknown Command "~msg); 1382 } 1383 1384 gs.tid_by_command_id[new_id] = tid; 1385 if (gs.command_line.terminal) 1386 { 1387 gs.command_line.command_in_focus_tid = tid; 1388 gs.command_line.command_in_focus_id = new_id; 1389 gs.keybar.input_mode = true; 1390 } 1391 }, 1392 (Tid tid, string msg) 1393 { 1394 //writefln("new_id=%s, tid=%s", new_id, tid); 1395 if (msg != "update terminal") 1396 { 1397 throw new Exception("Unknown Command "~msg); 1398 } 1399 1400 gs.command_line.last_redraw = 0; 1401 }, 1402 (shared(Throwable) exc) 1403 { 1404 throw exc; 1405 }, 1406 (shared ConsoleMessage smsg) 1407 { 1408 auto msg = cast(ConsoleMessage)(smsg); 1409 if (msg.message == "") 1410 msg.message = "Oops, empty message"; 1411 gs.messages ~= msg; 1412 writeln(msg.message); 1413 }, 1414 (Tid from, string path, string copy_to_mnt, bool move, MsgState command) 1415 { 1416 switch(command) 1417 { 1418 case MsgState.Send: 1419 auto copymapinfo = CopyMapInfo(path, null, move, from); 1420 gs.copy_map[copy_to_mnt] = copymapinfo; 1421 1422 foreach(i, scanner; gs.scanners) 1423 { 1424 //writefln("main.thread send(%s, %s, %s)", scanner, path, copy_to_mnt); 1425 send(scanner, path, copy_to_mnt, move); 1426 gs.copy_map[copy_to_mnt].sent[scanner]=true; 1427 } 1428 1429 //writefln("main.thread sent = %d", gs.copy_map[copy_to_mnt].sent.length); 1430 if (gs.copy_map[copy_to_mnt].sent.length == 0) 1431 { 1432 //writefln("main.thread send(resent) to %s (immidiately)", from); 1433 send(from, MsgState.Resent); 1434 } 1435 //writefln("copymapinfo.from = %s", copymapinfo.from); 1436 break; 1437 1438 case MsgState.Received: 1439 gs.copy_map[copy_to_mnt].sent.remove(from); 1440 //writefln("main.thread sent-- = %d", gs.copy_map[copy_to_mnt].sent.length); 1441 1442 if (gs.copy_map[copy_to_mnt].sent.length == 0) 1443 { 1444 //writefln("main.thread send(resent) to %s (after receiving received)", gs.copy_map[copy_to_mnt].from); 1445 send(gs.copy_map[copy_to_mnt].from, MsgState.Resent); 1446 } 1447 break; 1448 1449 case MsgState.Used: 1450 //writefln("notused: %s", gs.copy_map); 1451 if (! gs.copy_map.remove(copy_to_mnt) ) 1452 { 1453 writefln("Oh, no %s not found in copy map info", copy_to_mnt); 1454 } 1455 break; 1456 default: 1457 assert(false, "Unexpected MsgState"); 1458 } 1459 }, 1460 (string command, string arg) 1461 { 1462 if (command == "select") 1463 { 1464 auto pathmnt = PathMnt(gs.lsblk, arg); 1465 auto apply_rect = DRect(0, 0, 1024*1024, 1024*1024); 1466 auto drect = get_rectsize_for_path(gs, PathMnt(gs.lsblk, SL), arg, apply_rect); 1467 gs.selection_hash[pathmnt] = drect; 1468 gs.dirty = true; 1469 } 1470 else 1471 { 1472 writefln("Main thread. Unknown command %s", command); 1473 } 1474 }, 1475 (Variant v) 1476 { 1477 writefln("main.thread UNKNOWN MESSAGE: %s", v); 1478 } 1479 ) 1480 ) 1481 { 1482 } 1483 } 1484 1485 foreach(i, pid; gs.pids ) 1486 { 1487 auto dmd = tryWait(pid); 1488 if (dmd.terminated) 1489 gs.pids = gs.pids[0..i] ~ gs.pids[i+1..$]; 1490 } 1491 } 1492