Map<Path, PartitionDesc> sourcePathToPartitionInfo = source.getPathToPartitionInfo(); Map<String, Operator<? extends OperatorDesc>> sourceAliasToWork = source.getAliasToWork(); Map<String, PartitionDesc> sourceAliasToPartnInfo = source.getAliasToPartnInfo(); Map<String, PartitionDesc> targetAliasToPartnInfo = target.getAliasToPartnInfo();
@SuppressWarnings("rawtypes") private static Path createDummyFileForEmptyTable(JobConf job, MapWork work, Path hiveScratchDir, String alias) throws Exception { TableDesc tableDesc = work.getAliasToPartnInfo().get(alias).getTableDesc(); if (tableDesc.isNonNative()) { // if it does not need native storage, we can't create an empty file for it. return null; } Properties props = tableDesc.getProperties(); HiveOutputFormat outFileFormat = HiveFileFormatUtils.getHiveOutputFormat(job, tableDesc); Path newPath = createEmptyFile(hiveScratchDir, outFileFormat, job, props, false); LOG.info("Changed input file for alias {} to newPath", alias, newPath); // update the work LinkedHashMap<Path, ArrayList<String>> pathToAliases = work.getPathToAliases(); ArrayList<String> newList = new ArrayList<String>(1); newList.add(alias); pathToAliases.put(newPath, newList); work.setPathToAliases(pathToAliases); PartitionDesc pDesc = work.getAliasToPartnInfo().get(alias).clone(); work.addPathToPartitionInfo(newPath, pDesc); return newPath; }
if (rowLength == null) { LOG.debug("No table property in JobConf. Try to recover the table directly"); Map<String, PartitionDesc> partitionDescMap = Utilities.getMapRedWork(job).getMapWork().getAliasToPartnInfo(); for (String alias : Utilities.getMapRedWork(job).getMapWork().getAliasToPartnInfo().keySet()) { LOG.debug(format("the current alias: %s", alias)); rowLength = partitionDescMap.get(alias).getTableDesc().getProperties().getProperty(TD_ROW_LENGTH);
Map<Path, PartitionDesc> sourcePathToPartitionInfo = source.getPathToPartitionInfo(); Map<String, Operator<? extends OperatorDesc>> sourceAliasToWork = source.getAliasToWork(); Map<String, PartitionDesc> sourceAliasToPartnInfo = source.getAliasToPartnInfo(); Map<String, PartitionDesc> targetAliasToPartnInfo = target.getAliasToPartnInfo();
@SuppressWarnings("rawtypes") private static Path createDummyFileForEmptyTable(JobConf job, MapWork work, Path hiveScratchDir, String alias) throws Exception { TableDesc tableDesc = work.getAliasToPartnInfo().get(alias).getTableDesc(); if (tableDesc.isNonNative()) { // if it does not need native storage, we can't create an empty file for it. return null; } Properties props = tableDesc.getProperties(); HiveOutputFormat outFileFormat = HiveFileFormatUtils.getHiveOutputFormat(job, tableDesc); Path newPath = createEmptyFile(hiveScratchDir, outFileFormat, job, props, false); if (LOG.isInfoEnabled()) { LOG.info("Changed input file for alias " + alias + " to " + newPath); } // update the work LinkedHashMap<Path, ArrayList<String>> pathToAliases = work.getPathToAliases(); ArrayList<String> newList = new ArrayList<String>(); newList.add(alias); pathToAliases.put(newPath, newList); work.setPathToAliases(pathToAliases); PartitionDesc pDesc = work.getAliasToPartnInfo().get(alias).clone(); work.addPathToPartitionInfo(newPath, pDesc); return newPath; }
for (PartitionDesc pd: conf.getAliasToPartnInfo().values()) { if (!tableNameToConf.containsKey(pd.getTableName())) { tableNameToConf.put(pd.getTableName(), hconf);
for (PartitionDesc pd: conf.getAliasToPartnInfo().values()) { if (!tableNameToConf.containsKey(pd.getTableName())) { tableNameToConf.put(pd.getTableName(), hconf);
private void genSMBJoinWork(MapWork currWork, SMBMapJoinOperator smbJoinOp) { Map<String, PartitionDesc> aliasToPartitionInfo = currWork.getAliasToPartnInfo(); List<Path> removePaths = new ArrayList<>(); currWork.getAliasToPartnInfo().remove(alias); currWork.getAliasToWork().remove(alias); PartitionDesc partitionInfo = currWork.getAliasToPartnInfo().get(alias); if (fetchWork.getTblDir() != null) { currWork.mergeAliasedInput(alias, fetchWork.getTblDir(), partitionInfo);
TableDesc table = newWork.getMapWork().getAliasToPartnInfo().get(alias).getTableDesc(); fetchWork = new FetchWork(partDir, partDesc, table);
private void genSMBJoinWork(MapWork currWork, SMBMapJoinOperator smbJoinOp) { Map<String, PartitionDesc> aliasToPartitionInfo = currWork.getAliasToPartnInfo(); List<Path> removePaths = new ArrayList<>(); currWork.getAliasToPartnInfo().remove(alias); currWork.getAliasToWork().remove(alias); PartitionDesc partitionInfo = currWork.getAliasToPartnInfo().get(alias); if (fetchWork.getTblDir() != null) { currWork.mergeAliasedInput(alias, fetchWork.getTblDir(), partitionInfo);
TableDesc table = newWork.getMapWork().getAliasToPartnInfo().get(alias).getTableDesc(); fetchWork = new FetchWork(partDir, partDesc, table);
for (PartitionDesc part : mrwork.getAliasToPartnInfo().values()) { TableDesc tableDesc = part.getTableDesc(); Utilities.copyJobSecretToTableProperties(tableDesc);
public void initEmptyInputChildren(List<Operator<?>> children, Configuration hconf) throws SerDeException, Exception { setChildOperators(children); Map<String, Configuration> tableNameToConf = cloneConfsForNestedColPruning(hconf); for (Operator<?> child : children) { TableScanOperator tsOp = (TableScanOperator) child; StructObjectInspector soi = null; PartitionDesc partDesc = conf.getAliasToPartnInfo().get(tsOp.getConf().getAlias()); Configuration newConf = tableNameToConf.get(partDesc.getTableDesc().getTableName()); Deserializer serde = partDesc.getTableDesc().getDeserializer(); partDesc.setProperties(partDesc.getProperties()); MapOpCtx opCtx = new MapOpCtx(tsOp.getConf().getAlias(), child, partDesc); StructObjectInspector tableRowOI = (StructObjectInspector) serde.getObjectInspector(); initObjectInspector(newConf, opCtx, tableRowOI); soi = opCtx.rowObjectInspector; child.getParentOperators().add(this); childrenOpToOI.put(child, soi); } }
PartitionDesc partDesc = mWork.getAliasToPartnInfo().get(alias);
mapWork.getAliasToPartnInfo().size() == 1, "Requested to generate single split. Paths and fileStatuses are expected to be 1. " + "Got paths: " + paths.size() + " fileStatuses: " + fileStatuses.length); FileSplit fileSplit = new FileSplit(fileStatus.getPath(), 0, fileStatus.getLen(), hosts); String alias = mapWork.getAliases().get(0); PartitionDesc partDesc = mapWork.getAliasToPartnInfo().get(alias); String partIF = partDesc.getInputFileFormatClassName(); splits[0] = new HiveInputFormat.HiveInputSplit(fileSplit, partIF);
public void initEmptyInputChildren(List<Operator<?>> children, Configuration hconf) throws SerDeException, Exception { setChildOperators(children); Map<String, Configuration> tableNameToConf = cloneConfsForNestedColPruning(hconf); for (Operator<?> child : children) { TableScanOperator tsOp = (TableScanOperator) child; StructObjectInspector soi = null; PartitionDesc partDesc = conf.getAliasToPartnInfo().get(tsOp.getConf().getAlias()); Configuration newConf = tableNameToConf.get(partDesc.getTableDesc().getTableName()); Deserializer serde = partDesc.getTableDesc().getDeserializer(); partDesc.setProperties(partDesc.getProperties()); MapOpCtx opCtx = new MapOpCtx(tsOp.getConf().getAlias(), child, partDesc); StructObjectInspector tableRowOI = (StructObjectInspector) serde.getObjectInspector(); initObjectInspector(newConf, opCtx, tableRowOI); soi = opCtx.rowObjectInspector; child.getParentOperators().add(this); childrenOpToOI.put(child, soi); } }
PartitionDesc partDesc = mWork.getAliasToPartnInfo().get(alias);
PartitionDesc partitionDesc = new PartitionDesc(tableDescList.get(tags[j]), null); mapWork.addPathToPartitionInfo(path, partitionDesc); mapWork.getAliasToPartnInfo().put(alias, partitionDesc); mapWork.setName("Map " + GenSparkUtils.getUtils().getNextSeqNumber());
for (PartitionDesc part : mapWork.getAliasToPartnInfo().values()) { TableDesc tableDesc = part.getTableDesc(); Utilities.copyJobSecretToTableProperties(tableDesc);
for (PartitionDesc part : mrwork.getAliasToPartnInfo().values()) { TableDesc tableDesc = part.getTableDesc(); Utilities.copyJobSecretToTableProperties(tableDesc);