target.setPathToAliases(targetPathToAliases); target.setPathToPartitionInfo(targetPathToPartitionInfo);
/** * create a new plan and return. The pan won't contain the name to split * sample information in parse context. * * @return the new plan */ public static MapredWork getMapRedWorkFromConf(HiveConf conf) { MapredWork mrWork = new MapredWork(); MapWork work = mrWork.getMapWork(); boolean mapperCannotSpanPartns = conf.getBoolVar( HiveConf.ConfVars.HIVE_MAPPER_CANNOT_SPAN_MULTIPLE_PARTITIONS); work.setMapperCannotSpanPartns(mapperCannotSpanPartns); work.setPathToAliases(new LinkedHashMap<Path, ArrayList<String>>()); work.setPathToPartitionInfo(new LinkedHashMap<Path, PartitionDesc>()); work.setAliasToWork(new LinkedHashMap<String, Operator<? extends OperatorDesc>>()); return mrWork; }
target.setPathToAliases(targetPathToAliases); target.setPathToPartitionInfo(targetPathToPartitionInfo);
@Override public MapWork read(Kryo kryo, Input input, Class<MapWork> type) { MapWork mapWork = super.read(kryo, input, type); // The set methods in MapWork intern the any duplicate strings which is why we call them // during de-serialization mapWork.setPathToPartitionInfo(mapWork.getPathToPartitionInfo()); mapWork.setPathToAliases(mapWork.getPathToAliases()); return mapWork; } }
/** * create a new plan and return. The pan won't contain the name to split * sample information in parse context. * * @return the new plan */ public static MapredWork getMapRedWorkFromConf(HiveConf conf) { MapredWork mrWork = new MapredWork(); MapWork work = mrWork.getMapWork(); boolean mapperCannotSpanPartns = conf.getBoolVar( HiveConf.ConfVars.HIVE_MAPPER_CANNOT_SPAN_MULTIPLE_PARTITIONS); work.setMapperCannotSpanPartns(mapperCannotSpanPartns); work.setPathToAliases(new LinkedHashMap<Path, ArrayList<String>>()); work.setPathToPartitionInfo(new LinkedHashMap<Path, PartitionDesc>()); work.setAliasToWork(new LinkedHashMap<String, Operator<? extends OperatorDesc>>()); return mrWork; }
@SuppressWarnings("rawtypes") private static Path createDummyFileForEmptyTable(JobConf job, MapWork work, Path hiveScratchDir, String alias) throws Exception { TableDesc tableDesc = work.getAliasToPartnInfo().get(alias).getTableDesc(); if (tableDesc.isNonNative()) { // if it does not need native storage, we can't create an empty file for it. return null; } Properties props = tableDesc.getProperties(); HiveOutputFormat outFileFormat = HiveFileFormatUtils.getHiveOutputFormat(job, tableDesc); Path newPath = createEmptyFile(hiveScratchDir, outFileFormat, job, props, false); LOG.info("Changed input file for alias {} to newPath", alias, newPath); // update the work LinkedHashMap<Path, ArrayList<String>> pathToAliases = work.getPathToAliases(); ArrayList<String> newList = new ArrayList<String>(1); newList.add(alias); pathToAliases.put(newPath, newList); work.setPathToAliases(pathToAliases); PartitionDesc pDesc = work.getAliasToPartnInfo().get(alias).clone(); work.addPathToPartitionInfo(newPath, pDesc); return newPath; }
@Test public void testGetAndSetConsistency() { MapWork mw = new MapWork(); LinkedHashMap<Path, ArrayList<String>> pathToAliases = new LinkedHashMap<>(); pathToAliases.put(new Path("p0"), Lists.newArrayList("a1", "a2")); mw.setPathToAliases(pathToAliases); LinkedHashMap<Path, ArrayList<String>> pta = mw.getPathToAliases(); assertEquals(pathToAliases, pta); }
@SuppressWarnings("rawtypes") private static Path createDummyFileForEmptyTable(JobConf job, MapWork work, Path hiveScratchDir, String alias) throws Exception { TableDesc tableDesc = work.getAliasToPartnInfo().get(alias).getTableDesc(); if (tableDesc.isNonNative()) { // if it does not need native storage, we can't create an empty file for it. return null; } Properties props = tableDesc.getProperties(); HiveOutputFormat outFileFormat = HiveFileFormatUtils.getHiveOutputFormat(job, tableDesc); Path newPath = createEmptyFile(hiveScratchDir, outFileFormat, job, props, false); if (LOG.isInfoEnabled()) { LOG.info("Changed input file for alias " + alias + " to " + newPath); } // update the work LinkedHashMap<Path, ArrayList<String>> pathToAliases = work.getPathToAliases(); ArrayList<String> newList = new ArrayList<String>(); newList.add(alias); pathToAliases.put(newPath, newList); work.setPathToAliases(pathToAliases); PartitionDesc pDesc = work.getAliasToPartnInfo().get(alias).clone(); work.addPathToPartitionInfo(newPath, pDesc); return newPath; }
mapWork.setPathToAliases(pathToAliasTable);
mockPartitionDesc).getOutputFileFormatClass(); mapWork1.setPathToAliases(new LinkedHashMap<>( ImmutableMap.of(nonExistentPath1, Lists.newArrayList(alias1Name)))); mapWork1.setAliasToWork(new LinkedHashMap<>( ImmutableMap.of(nonExistentPath1, mockPartitionDesc))); mapWork2.setPathToAliases(new LinkedHashMap<>( ImmutableMap.of(nonExistentPath2, Lists.newArrayList(alias2Name)))); mapWork2.setAliasToWork(new LinkedHashMap<>(
mapWork.setPathToAliases(pathToAliasTable); mapWork.setPathToPartitionInfo(pathToPartitionInfo);
mapWork.setPathToAliases(aliasMap); mapWork.setPathToPartitionInfo(partMap);
pathMap.put(new Path("foo"), aliasList); mws[0].setPathToAliases(pathMap); mws[1].setPathToAliases(pathMap);
mrwork.getMapWork().setPathToAliases(pa); mrwork.getMapWork().setPathToPartitionInfo(pt); mrwork.getMapWork().setAliasToWork(ao);
mrwork.getMapWork().setPathToAliases(pathToAliases); mrwork.getMapWork().setPathToPartitionInfo(pathToPartitionInfo); mrwork.getMapWork().setAliasToWork(aliasToWork);
@SuppressWarnings("rawtypes") private static Path createDummyFileForEmptyTable(JobConf job, MapWork work, Path hiveScratchDir, String alias, int sequenceNumber) throws Exception { TableDesc tableDesc = work.getAliasToPartnInfo().get(alias).getTableDesc(); if (tableDesc.isNonNative()) { // if this isn't a hive table we can't create an empty file for it. return null; } Properties props = tableDesc.getProperties(); HiveOutputFormat outFileFormat = HiveFileFormatUtils.getHiveOutputFormat(job, tableDesc); Path newPath = createEmptyFile(hiveScratchDir, outFileFormat, job, sequenceNumber, props, false); if (LOG.isInfoEnabled()) { LOG.info("Changed input file for alias " + alias + " to " + newPath); } // update the work LinkedHashMap<String, ArrayList<String>> pathToAliases = work.getPathToAliases(); ArrayList<String> newList = new ArrayList<String>(); newList.add(alias); pathToAliases.put(newPath.toUri().toString(), newList); work.setPathToAliases(pathToAliases); LinkedHashMap<String, PartitionDesc> pathToPartitionInfo = work.getPathToPartitionInfo(); PartitionDesc pDesc = work.getAliasToPartnInfo().get(alias).clone(); pathToPartitionInfo.put(newPath.toUri().toString(), pDesc); work.setPathToPartitionInfo(pathToPartitionInfo); return newPath; }
/** * create a new plan and return. The pan won't contain the name to split * sample information in parse context. * * @return the new plan */ public static MapredWork getMapRedWorkFromConf(HiveConf conf) { MapredWork mrWork = new MapredWork(); MapWork work = mrWork.getMapWork(); boolean mapperCannotSpanPartns = conf.getBoolVar( HiveConf.ConfVars.HIVE_MAPPER_CANNOT_SPAN_MULTIPLE_PARTITIONS); work.setMapperCannotSpanPartns(mapperCannotSpanPartns); work.setPathToAliases(new LinkedHashMap<String, ArrayList<String>>()); work.setPathToPartitionInfo(new LinkedHashMap<String, PartitionDesc>()); work.setAliasToWork(new LinkedHashMap<String, Operator<? extends OperatorDesc>>()); work.setHadoopSupportsSplittable( conf.getBoolVar(HiveConf.ConfVars.HIVE_COMBINE_INPUT_FORMAT_SUPPORTS_SPLITTABLE)); return mrWork; }
pathToAliases.remove(strPath); work.setPathToAliases(pathToAliases);