Skip to content

Commit

Permalink
Locations are built when adding splits
Browse files Browse the repository at this point in the history
  • Loading branch information
Roberto R. Expósito committed Apr 22, 2022
1 parent 3108a75 commit 99ad370
Showing 1 changed file with 18 additions and 24 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -22,9 +22,9 @@
import java.io.DataOutput;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.stream.Collectors;
import java.util.stream.Stream;

import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Writable;
Expand All @@ -37,15 +37,18 @@ public class PairedEndInputSplit extends FileSplit implements Writable {

private static final int LENGTH = 2;

private int fill = 0;
private long totsize = 0L;
private int fill;
private long totsize;
private FileSplit[] splits;
private List<List<String>> allHosts;
private String[] hosts;

public PairedEndInputSplit() {
splits = new FileSplit[LENGTH];
allHosts = new ArrayList<List<String>>(LENGTH);
hosts = null;
fill = 0;
totsize = 0L;

for (int i = 0; i < LENGTH; i++)
allHosts.add(new ArrayList<String>());
Expand All @@ -63,9 +66,6 @@ public void add(FileSplit s) throws IOException, InterruptedException {
throw new IOException("Too many splits");
}

splits[fill] = s;
totsize += s.getLength();

String[] hints = s.getLocations();

if (hints != null && hints.length > 0) {
Expand All @@ -74,14 +74,23 @@ public void add(FileSplit s) throws IOException, InterruptedException {
}
}

fill++;
splits[fill++] = s;
totsize += s.getLength();

if (fill == LENGTH) {
List<String> intersect = allHosts.get(0).stream()
.filter(allHosts.get(1)::contains)
.collect(Collectors.toList());

hosts = intersect.toArray(new String[intersect.size()]);
if (intersect.size() > 0) {
hosts = intersect.toArray(new String[intersect.size()]);
} else {
List<String> union = Stream.concat(allHosts.get(0).stream(), allHosts.get(1).stream())
.distinct()
.collect(Collectors.toList());

hosts = union.toArray(new String[union.size()]);
}
}
}

Expand Down Expand Up @@ -140,22 +149,7 @@ public String toString() {
*/
@Override
public String[] getLocations() throws IOException {
if (hosts.length > 0)
return hosts;

HashSet<String> hostsSet = new HashSet<String>();

for (int i = 0; i < splits.length; i++) {
String[] hints = splits[i].getLocations();

if (hints != null && hints.length > 0) {
for (String host : hints) {
hostsSet.add(host);
}
}
}

return hostsSet.toArray(new String[hostsSet.size()]);
return hosts;
}

/**
Expand Down

0 comments on commit 99ad370

Please sign in to comment.