public void initialize(MapWork work, JobConf jobConf) throws SerDeException { Map<String, SourceInfo> columnMap = new HashMap<String, SourceInfo>(); Set<String> sourceWorkIds = work.getEventSourceTableDescMap().keySet(); for (String id : sourceWorkIds) { List<TableDesc> tables = work.getEventSourceTableDescMap().get(id); List<String> columnNames = work.getEventSourceColumnNameMap().get(id); List<ExprNodeDesc> partKeyExprs = work.getEventSourcePartKeyExprMap().get(id); Iterator<String> cit = columnNames.iterator(); Iterator<ExprNodeDesc> pit = partKeyExprs.iterator(); for (TableDesc t : tables) { String columnName = cit.next(); ExprNodeDesc partKeyExpr = pit.next(); SourceInfo si = new SourceInfo(t, partKeyExpr, columnName, jobConf); if (!sourceInfoMap.containsKey(id)) { sourceInfoMap.put(id, new ArrayList<SourceInfo>()); } sourceInfoMap.get(id).add(si); // We could have multiple sources restrict the same column, need to take // the union of the values in that case. if (columnMap.containsKey(columnName)) { si.values = columnMap.get(columnName).values; } columnMap.put(columnName, si); } } }
public void initialize(MapWork work, JobConf jobConf) throws SerDeException { Map<String, SourceInfo> columnMap = new HashMap<String, SourceInfo>(); Set<String> sourceWorkIds = work.getEventSourceTableDescMap().keySet(); List<TableDesc> tables = work.getEventSourceTableDescMap().get(id);
Set<TableDesc> tableDescs1 = new HashSet<>(first.getEventSourceTableDescMap().get(source)); Set<TableDesc> tableDescs2 = new HashSet<>(second.getEventSourceTableDescMap().get(source)); if (!tableDescs1.equals(tableDescs2)) { return false;
/** * Add this DPP sink as a pruning source for the target MapWork. It means the DPP sink's output * will be used to prune a certain partition in the MapWork. The MapWork's event source maps will * be updated to remember the DPP sink's unique ID and corresponding target columns. */ public void addAsSourceEvent(MapWork mapWork, ExprNodeDesc partKey, String columnName, String columnType) { String sourceId = getUniqueId(); SparkPartitionPruningSinkDesc conf = getConf(); // store table descriptor in map-targetWork List<TableDesc> tableDescs = mapWork.getEventSourceTableDescMap().computeIfAbsent(sourceId, v -> new ArrayList<>()); tableDescs.add(conf.getTable()); // store partition key expr in map-targetWork List<ExprNodeDesc> partKeys = mapWork.getEventSourcePartKeyExprMap().computeIfAbsent(sourceId, v -> new ArrayList<>()); partKeys.add(partKey); // store column name in map-targetWork List<String> columnNames = mapWork.getEventSourceColumnNameMap().computeIfAbsent(sourceId, v -> new ArrayList<>()); columnNames.add(columnName); List<String> columnTypes = mapWork.getEventSourceColumnTypeMap().computeIfAbsent(sourceId, v -> new ArrayList<>()); columnTypes.add(columnType); }
Map<String, SourceInfo> columnMap = new HashMap<String, SourceInfo>(); Set<String> sources = work.getEventSourceTableDescMap().keySet(); numEventsSeenPerSource.put(s, new MutableInt(0)); List<TableDesc> tables = work.getEventSourceTableDescMap().get(s);
SparkPartitionPruningSinkDesc conf = getConf(); List<TableDesc> tableDescs = mapWork.getEventSourceTableDescMap().get(sourceId); if (tableDescs != null) { tableDescs.remove(conf.getTable()); if (tableDescs.isEmpty()) { mapWork.getEventSourceTableDescMap().remove(sourceId);
Map<String, SourceInfo> columnMap = new HashMap<String, SourceInfo>(); Set<String> sources = work.getEventSourceTableDescMap().keySet(); numEventsSeenPerSource.put(s, new MutableInt(0)); List<TableDesc> tables = work.getEventSourceTableDescMap().get(s);
if (!work.getEventSourceTableDescMap().containsKey(sourceName)) { work.getEventSourceTableDescMap().put(sourceName, new LinkedList<TableDesc>()); List<TableDesc> tables = work.getEventSourceTableDescMap().get(sourceName); tables.add(event.getConf().getTable());
if (!work.getEventSourceTableDescMap().containsKey(sourceName)) { work.getEventSourceTableDescMap().put(sourceName, new LinkedList<TableDesc>()); List<TableDesc> tables = work.getEventSourceTableDescMap().get(sourceName); tables.add(event.getConf().getTable());
doReturn(tableMap).when(mapWork).getEventSourceTableDescMap(); doReturn(columnMap).when(mapWork).getEventSourceColumnNameMap(); doReturn(exprMap).when(mapWork).getEventSourcePartKeyExprMap();
if (!targetWork.getEventSourceTableDescMap().containsKey(sourceId)) { targetWork.getEventSourceTableDescMap().put(sourceId, new LinkedList<TableDesc>()); List<TableDesc> tables = targetWork.getEventSourceTableDescMap().get(sourceId); tables.add(pruningSink.getConf().getTable());
Map<String, SourceInfo> columnMap = new HashMap<String, SourceInfo>(); Set<String> sources = work.getEventSourceTableDescMap().keySet(); numEventsSeenPerSource.put(s, new MutableInt(0)); List<TableDesc> tables = work.getEventSourceTableDescMap().get(s);
if (!work.getEventSourceTableDescMap().containsKey(sourceName)) { work.getEventSourceTableDescMap().put(sourceName, new LinkedList<TableDesc>()); List<TableDesc> tables = work.getEventSourceTableDescMap().get(sourceName); tables.add(event.getConf().getTable());