参考链接https://blog.csdn.net/qq1010234991/article/details/87864654默认配置propertynamemapreduce.job.reduce.slowstart.completedmaps/namevalue0.05/valuedescriptionFraction of the numberofmapsinthe job which should be complete before reduces are scheduledforthe job./description/propertypropertynamemapreduce.reduce.shuffle.parallelcopies/namevalue5/valuedescriptionThedefaultnumber of parallel transfers runbyreduce duringthecopy(shuffle)phase./description/propertyAccumulatingReducerpublicclassAccumulatingReducerextends MapReduceBaseimplementsReducerText,Text,Text,Text{staticfinalStringVALUE_TYPE_LONGl:;staticfinalStringVALUE_TYPE_FLOATf:;staticfinalStringVALUE_TYPE_STRINGs:;privatestaticfinalLoggerLOGLoggerFactory.getLogger(AccumulatingReducer.class);protectedStringhostName;publicAccumulatingReducer(){try{hostNamejava.net.InetAddress.getLocalHost().getHostName();}catch(Exceptione){hostNamelocalhost;}LOG.info(Starting AccumulatingReducer on hostName);}publicvoidreduce(Textkey,IteratorTextvalues,OutputCollectorText,Textoutput,Reporterreporter)throwsIOException{Stringfieldkey.toString();reporter.setStatus(starting field ::host hostName);// concatenate stringsif(field.startsWith(VALUE_TYPE_STRING)){StringBuffersSumnewStringBuffer();while(values.hasNext())sSum.append(values.next().toString()).append(;);output.collect(key,newText(sSum.toString()));reporter.setStatus(finished field ::host hostName);return;}// sum long valuesif(field.startsWith(VALUE_TYPE_FLOAT)){floatfSum0;while(values.hasNext())fSumFloat.parseFloat(values.next().toString());output.collect(key,newText(String.valueOf(fSum)));reporter.setStatus(finished field ::host hostName);return;}// sum long valuesif(field.startsWith(VALUE_TYPE_LONG)){longlSum0;while(values.hasNext()){lSumLong.parseLong(values.next().toString());}output.collect(key,newText(String.valueOf(lSum)));}reporter.setStatus(finished field ::host hostName);}}mapreduce.job.reduce.slowstart.completedmaps本质上是启动多个线程// Start the map-output fetcher threadsbooleanisLocallocalMapFiles!null;finalintnumFetchersisLocal?1:jobConf.getInt(MRJobConfig.SHUFFLE_PARALLEL_COPIES,5);FetcherK,V[]fetchersnewFetcher[numFetchers];if(isLocal){fetchers[0]newLocalFetcherK,V(jobConf,reduceId,scheduler,merger,reporter,metrics,this,reduceTask.getShuffleSecret(),localMapFiles);fetchers[0].start();}else{for(inti0;inumFetchers;i){fetchers[i]newFetcherK,V(jobConf,reduceId,scheduler,merger,reporter,metrics,this,reduceTask.getShuffleSecret());fetchers[i].start();}}classFetcherK,VextendsThread{privatestaticfinalLoggerLOGLoggerFactory.getLogger(Fetcher.class);Metrics2025-12-02 11:53:59,162 INFO mapreduce.Job: Counters: 54 File System Counters FILE: Number of bytes read9957 FILE: Number of bytes written101500878 FILE: Number of read operations0 FILE: Number of large read operations0 FILE: Number of write operations0 HDFS: Number of bytes read1509949525460 HDFS: Number of bytes written90 HDFS: Number of read operations1805 HDFS: Number of large read operations0 HDFS: Number of write operations2 HDFS: Number of bytes read erasure-coded0 Job Counters Launched map tasks360 Launched reduce tasks1 Data-local map tasks360 Total time spent by all maps in occupied slots (ms)20423063 Total time spent by all reduces in occupied slots (ms)45884 Total time spent by all map tasks (ms)20423063 Total time spent by all reduce tasks (ms)22942 Total vcore-milliseconds taken by all map tasks20423063 Total vcore-milliseconds taken by all reduce tasks22942 Total megabyte-milliseconds taken by all map tasks41826433024 Total megabyte-milliseconds taken by all reduce tasks93970432 Map-Reduce Framework Map input records360 Map output records1800 Map output bytes24223 Map output materialized bytes36847 Input split bytes44530 Combine input records0 Combine output records0 Reduce input groups5 Reduce shuffle bytes36847 Reduce input records1800 Reduce output records5 Spilled Records3600 Shuffled Maps 360 Failed Shuffles0 Merged Map outputs360 GC time elapsed (ms)47808 CPU time spent (ms)6328140 Physical memory (bytes) snapshot140393537536 Virtual memory (bytes) snapshot634706259968 Total committed heap usage (bytes)381780230144 Peak Map Physical memory (bytes)418402304 Peak Map Virtual memory (bytes)2308349952 Peak Reduce Physical memory (bytes)446230528 Peak Reduce Virtual memory (bytes)1797619712HDFS: Number of bytes read1509949525460 // 1.51TB数据读取 HDFS: Number of bytes written90 // 仅90字节写入可能是结果文件 HDFS: Number of read operations1805 // 1805次读取操作FILE: Number of bytes read9957 // 本地读取很少 FILE: Number of bytes written101500878 // 101MB本地写入中间结果Launched map tasks360 // 360个Map任务 Launched reduce tasks1 // 1个Reduce任务 Data-local map tasks360 // 100%数据本地性优秀// 输入输出记录流 Map input records360 // Map输入360条记录 Map output records1800 // Map输出1800条记录5倍扩张 Reduce input records1800 // Reduce输入1800条记录 Reduce output records5 // Reduce输出5条记录高度聚合 // 数据大小变化 Map output bytes24223 // Map输出24KB Map output materialized bytes36847 // 物化输出37KB Reduce shuffle bytes36847 // Shuffle数据37KBmap 并发确认