partitionDS

Partition a dataset into pieces for handling by multiple MPI ranks

xxxxxxxxxx
 
md"""
# partitionDS
​
Partition a dataset into pieces for handling by multiple MPI ranks
"""

11.9 μs

xxxxxxxxxx
 
begin
    using Pkg
    Pkg.activate(".")
    
    using IRMA
    using Plots
    using PlutoUI
end

14.1 s

Enter cell code...xxxxxxxxxx
 
​

540 ns

UnitRange{Int64}1

1:15625

15626:31250

31251:46875

46876:62500

62501:78125

78126:93750

93751:109375

109376:125000

125001:140625

140626:156250

156251:171875

171876:187500

187501:203125

203126:218750

218751:234375

234376:250000

250001:265625

265626:281250

281251:296875

296876:312500

312501:328125

328126:343750

343751:359375

359376:375000

375001:390625

390626:406250

406251:421875

421876:437500

437501:453125

453126:468750

468751:484375

484376:500000

500001:515625

515626:531250

531251:546875

546876:562500

562501:578125

578126:593750

593751:609375

609376:625000

843751:859375

859376:875000

875001:890625

890626:906250

906251:921875

921876:937500

937501:953125

953126:968750

968751:984375

984376:1000000

xxxxxxxxxx
 
# Play with 1 million rows and 64 MPI ranks
r1 = partitionDS(1_000_000, 64)

5.1 ms

Int641

xxxxxxxxxx
 
l = length.(r1)

49 ms

15625.0

xxxxxxxxxx
 
1_000_000/64

13 μs

UnitRange{Int64}1

1:10417

10418:20834

20835:31251

31252:41668

41669:52085

52086:62502

62503:72919

72920:83336

83337:93753

93754:104170

104171:114587

114588:125004

125005:135421

135422:145838

145839:156255

156256:166672

166673:177089

177090:187506

187507:197923

197924:208340

208341:218757

218758:229174

229175:239591

239592:250008

250009:260425

260426:270842

270843:281259

281260:291676

291677:302093

302094:312510

312511:322927

322928:333344

333345:343761

343762:354178

354179:364595

364596:375012

375013:385429

385430:395846

395847:406263

406264:416680

895841:906256

906257:916672

916673:927088

927089:937504

937505:947920

947921:958336

958337:968752

968753:979168

979169:989584

989585:1000000

xxxxxxxxxx
 
# Let's try 96 ranks
r2 = partitionDS(1_000_000, 96)

5.5 μs

Int641

xxxxxxxxxx
 
l2 = length.(r2)

9.2 μs

xxxxxxxxxx
 
histogram(l2, legend = nothing)

5.3 s

xxxxxxxxxx
 
@bind nRanks Slider(1:200)

34.9 ms

Int641

175

176

177

178

179

180

181

182

183

184

xxxxxxxxxx
 
r3 = partitionDS(1_000_000, nRanks) ; l3=length.(r3)

14 μs

xxxxxxxxxx
 
histogram(l3, legend=nothing, xlab="Length of partitions for $nRanks ranks")

1.4 ms

Julia Version 1.5.2
Commit 539f3ce943 (2020-09-23 23:17 UTC)
Platform Info:
  OS: macOS (x86_64-apple-darwin18.7.0)
  uname: Darwin 19.6.0 Darwin Kernel Version 19.6.0: Thu Jun 18 20:49:00 PDT 2020; root:xnu-6153.141.1~1/RELEASE_X86_64 x86_64 i386
  CPU: Intel(R) Core(TM) i9-8950HK CPU @ 2.90GHz: 
                 speed         user         nice          sys         idle          irq
       #1-12  2900 MHz    4295292 s          0 s    2985030 s   33242993 s          0 s
       
  Memory: 32.0 GB (3374.265625 MB free)
  Uptime: 636297.0 sec
  Load Avg:  3.69384765625  3.14111328125  3.44189453125
  WORD_SIZE: 64
  LIBM: libopenlibm
  LLVM: libLLVM-9.0.1 (ORCJIT, skylake)
Environment:
  JULIA_REVISE_WORKER_ONLY = 1
  PATH = /Users/lyon/.julia/bin:/Users/lyon/.julia/bin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin:/Library/TeX/texbin:/opt/X11/bin:/Library/Apple/usr/bin:/Users/lyon/anaconda3/bin:/Users/lyon/anaconda3/condabin
  HOME = /Users/lyon
  XPC_FLAGS = 0x0
  TERM = xterm-256color
  FPATH = /Users/lyon/.julia/completions:/Users/lyon/.oh-my-zsh/plugins/docker:/Users/lyon/.oh-my-zsh/plugins/git:/Users/lyon/.oh-my-zsh/functions:/Users/lyon/.oh-my-zsh/completions:/usr/local/share/zsh/site-functions:/usr/share/zsh/site-functions:/usr/share/zsh/5.7.1/functions
  GKS_FONTPATH = /Users/lyon/.julia/packages/GR/BwGt2/src/../deps/gr
  GKS_FILEPATH = /var/folders/wj/42qx_p_5489__2px56437jg00000gp/T/jl_YgO8Le.svg

xxxxxxxxxx
 
with_terminal() do
    versioninfo(verbose=true)
end

1.2 s