本地跑Mapreduce程序的相关配置

本地跑MapReduce程序需要配置的代码

为了在本地运行MapReduce程序，需要加如下的东西
在项目中创建一个如图所示的包：org.apache.hadoop.io.nativeio，并在该包下面创建一个名为：NativeIO的类（注意：名字不能写错），在该类中加入如下的代码：
在这里插入图片描述

/*** Licensed to the Apache Software Foundation (ASF) under one* or more contributor license agreements.  See the NOTICE file* distributed with this work for additional information* regarding copyright ownership.  The ASF licenses this file* to you under the Apache License, Version 2.0 (the* "License"); you may not use this file except in compliance* with the License.  You may obtain a copy of the License at**     http://www.apache.org/licenses/LICENSE-2.0** Unless required by applicable law or agreed to in writing, software* distributed under the License is distributed on an "AS IS" BASIS,* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.* See the License for the specific language governing permissions and* limitations under the License.*/
package org.apache.hadoop.io.nativeio;import com.google.common.annotations.VisibleForTesting;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.fs.HardLink;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.io.SecureIOUtils.AlreadyExistsException;
import org.apache.hadoop.util.NativeCodeLoader;
import org.apache.hadoop.util.PerformanceAdvisory;
import org.apache.hadoop.util.Shell;
import sun.misc.Unsafe;import java.io.*;
import java.lang.reflect.Field;
import java.nio.ByteBuffer;
import java.nio.MappedByteBuffer;
import java.nio.channels.FileChannel;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;/*** JNI wrappers for various native IO-related calls not available in Java. These* functions should generally be used alongside a fallback to another more* portable mechanism.*/
@InterfaceAudience.Private
@InterfaceStability.Unstable
public class NativeIO {public static class POSIX {// Flags for open() call from bits/fcntl.hpublic static final int O_RDONLY = 00;public static final int O_WRONLY = 01;public static final int O_RDWR = 02;public static final int O_CREAT = 0100;public static final int O_EXCL = 0200;public static final int O_NOCTTY = 0400;public static final int O_TRUNC = 01000;public static final int O_APPEND = 02000;public static final int O_NONBLOCK = 04000;public static final int O_SYNC = 010000;public static final int O_ASYNC = 020000;public static final int O_FSYNC = O_SYNC;public static final int O_NDELAY = O_NONBLOCK;// Flags for posix_fadvise() from bits/fcntl.h/* No further special treatment. */public static final int POSIX_FADV_NORMAL = 0;/* Expect random page references. */public static final int POSIX_FADV_RANDOM = 1;/* Expect sequential page references. */public static final int POSIX_FADV_SEQUENTIAL = 2;/* Will need these pages. */public static final int POSIX_FADV_WILLNEED = 3;/* Don't need these pages. */public static final int POSIX_FADV_DONTNEED = 4;/* Data will be accessed once. */public static final int POSIX_FADV_NOREUSE = 5;/** Wait upon writeout of all pages in the range before performing the write.*/public static final int SYNC_FILE_RANGE_WAIT_BEFORE = 1;/** Initiate writeout of all those dirty pages in the range which are not* presently under writeback.*/public static final int SYNC_FILE_RANGE_WRITE = 2;/** Wait upon writeout of all pages in the range after performing the write.*/public static final int SYNC_FILE_RANGE_WAIT_AFTER = 4;private static final Log LOG = LogFactory.getLog(NativeIO.class);private static boolean nativeLoaded = false;private static boolean fadvisePossible = true;private static boolean syncFileRangePossible = true;static final String WORKAROUND_NON_THREADSAFE_CALLS_KEY = "hadoop.workaround.non.threadsafe.getpwuid";static final boolean WORKAROUND_NON_THREADSAFE_CALLS_DEFAULT = true;private static long cacheTimeout = -1;private static CacheManipulator cacheManipulator = new CacheManipulator();public static CacheManipulator getCacheManipulator() {return cacheManipulator;}public static void setCacheManipulator(CacheManipulator cacheManipulator) {POSIX.cacheManipulator = cacheManipulator;}/*** Used to manipulate the operating system cache.*/@VisibleForTestingpublic static class CacheManipulator {public void mlock(String identifier, ByteBuffer buffer, long len) throws IOException {POSIX.mlock(buffer, len);}public long getMemlockLimit() {return NativeIO.getMemlockLimit();}public long getOperatingSystemPageSize() {return NativeIO.getOperatingSystemPageSize();}public void posixFadviseIfPossible(String identifier, FileDescriptor fd, long offset, long len, int flags)throws NativeIOException {NativeIO.POSIX.posixFadviseIfPossible(identifier, fd, offset, len, flags);}public boolean verifyCanMlock() {return NativeIO.isAvailable();}}/*** A CacheManipulator used for testing which does not actually call mlock. This* allows many tests to be run even when the operating system does not allow* mlock, or only allows limited mlocking.*/@VisibleForTestingpublic static class NoMlockCacheManipulator extends CacheManipulator {public void mlock(String identifier, ByteBuffer buffer, long len) throws IOException {LOG.info("mlocking " + identifier);}public long getMemlockLimit() {return 1125899906842624L;}public long getOperatingSystemPageSize() {return 4096;}public boolean verifyCanMlock() {return true;}}static {if (NativeCodeLoader.isNativeCodeLoaded()) {try {Configuration conf = new Configuration();workaroundNonThreadSafePasswdCalls = conf.getBoolean(WORKAROUND_NON_THREADSAFE_CALLS_KEY,WORKAROUND_NON_THREADSAFE_CALLS_DEFAULT);initNative();nativeLoaded = true;cacheTimeout = conf.getLong(CommonConfigurationKeys.HADOOP_SECURITY_UID_NAME_CACHE_TIMEOUT_KEY,CommonConfigurationKeys.HADOOP_SECURITY_UID_NAME_CACHE_TIMEOUT_DEFAULT) * 1000;LOG.debug("Initialized cache for IDs to User/Group mapping with a " + " cache timeout of "+ cacheTimeout / 1000 + " seconds.");} catch (Throwable t) {// This can happen if the user has an older version of libhadoop.so// installed - in this case we can continue without native IO// after warningPerformanceAdvisory.LOG.debug("Unable to initialize NativeIO libraries", t);}}}/*** Return true if the JNI-based native IO extensions are available.*/public static boolean isAvailable() {return NativeCodeLoader.isNativeCodeLoaded() && nativeLoaded;}private static void assertCodeLoaded() throws IOException {if (!isAvailable()) {throw new IOException("NativeIO was not loaded");}}/** Wrapper around open(2) */public static native FileDescriptor open(String path, int flags, int mode) throws IOException;/** Wrapper around fstat(2) */private static native Stat fstat(FileDescriptor fd) throws IOException;/** Native chmod implementation. On UNIX, it is a wrapper around chmod(2) */private static native void chmodImpl(String path, int mode) throws IOException;public static void chmod(String path, int mode) throws IOException {if (!Shell.WINDOWS) {chmodImpl(path, mode);} else {try {chmodImpl(path, mode);} catch (NativeIOException nioe) {if (nioe.getErrorCode() == 3) {throw new NativeIOException("No such file or directory", Errno.ENOENT);} else {LOG.warn(String.format("NativeIO.chmod error (%d): %s", nioe.getErrorCode(), nioe.getMessage()));throw new NativeIOException("Unknown error", Errno.UNKNOWN);}}}}/** Wrapper around posix_fadvise(2) */static native void posix_fadvise(FileDescriptor fd, long offset, long len, int flags) throws NativeIOException;/** Wrapper around sync_file_range(2) */static native void sync_file_range(FileDescriptor fd, long offset, long nbytes, int flags)throws NativeIOException;/*** Call posix_fadvise on the given file descriptor. See the manpage for this* syscall for more information. On systems where this call is not available,* does nothing.** @throws NativeIOException*             if there is an error with the syscall*/static void posixFadviseIfPossible(String identifier, FileDescriptor fd, long offset, long len, int flags)throws NativeIOException {if (nativeLoaded && fadvisePossible) {try {posix_fadvise(fd, offset, len, flags);} catch (UnsupportedOperationException uoe) {fadvisePossible = false;} catch (UnsatisfiedLinkError ule) {fadvisePossible = false;}}}/*** Call sync_file_range on the given file descriptor. See the manpage for this* syscall for more information. On systems where this call is not available,* does nothing.** @throws NativeIOException*             if there is an error with the syscall*/public static void syncFileRangeIfPossible(FileDescriptor fd, long offset, long nbytes, int flags)throws NativeIOException {if (nativeLoaded && syncFileRangePossible) {try {sync_file_range(fd, offset, nbytes, flags);} catch (UnsupportedOperationException uoe) {syncFileRangePossible = false;} catch (UnsatisfiedLinkError ule) {syncFileRangePossible = false;}}}static native void mlock_native(ByteBuffer buffer, long len) throws NativeIOException;/*** Locks the provided direct ByteBuffer into memory, preventing it from swapping* out. After a buffer is locked, future accesses will not incur a page fault.** See the mlock(2) man page for more information.** @throws NativeIOException*/static void mlock(ByteBuffer buffer, long len) throws IOException {assertCodeLoaded();if (!buffer.isDirect()) {throw new IOException("Cannot mlock a non-direct ByteBuffer");}mlock_native(buffer, len);}/*** Unmaps the block from memory. See munmap(2).** There isn't any portable way to unmap a memory region in Java. So we use the* sun.nio method here. Note that unmapping a memory region could cause crashes* if code continues to reference the unmapped code. However, if we don't* manually unmap the memory, we are dependent on the finalizer to do it, and we* have no idea when the finalizer will run.** @param buffer*            The buffer to unmap.*/public static void munmap(MappedByteBuffer buffer) {if (buffer instanceof sun.nio.ch.DirectBuffer) {sun.misc.Cleaner cleaner = ((sun.nio.ch.DirectBuffer) buffer).cleaner();cleaner.clean();}}/** Linux only methods used for getOwner() implementation */private static native long getUIDforFDOwnerforOwner(FileDescriptor fd) throws IOException;private static native String getUserName(long uid) throws IOException;/*** Result type of the fstat call*/public static class Stat {private int ownerId, groupId;private String owner, group;private int mode;// Mode constantspublic static final int S_IFMT = 0170000; /* type of file */public static final int S_IFIFO = 0010000; /* named pipe (fifo) */public static final int S_IFCHR = 0020000; /* character special */public static final int S_IFDIR = 0040000; /* directory */public static final int S_IFBLK = 0060000; /* block special */public static final int S_IFREG = 0100000; /* regular */public static final int S_IFLNK = 0120000; /* symbolic link */public static final int S_IFSOCK = 0140000; /* socket */public static final int S_IFWHT = 0160000; /* whiteout */public static final int S_ISUID = 0004000; /* set user id on execution */public static final int S_ISGID = 0002000; /* set group id on execution */public static final int S_ISVTX = 0001000; /* save swapped text even after use */public static final int S_IRUSR = 0000400; /* read permission, owner */public static final int S_IWUSR = 0000200; /* write permission, owner */public static final int S_IXUSR = 0000100; /* execute/search permission, owner */Stat(int ownerId, int groupId, int mode) {this.ownerId = ownerId;this.groupId = groupId;this.mode = mode;}Stat(String owner, String group, int mode) {if (!Shell.WINDOWS) {this.owner = owner;} else {this.owner = stripDomain(owner);}if (!Shell.WINDOWS) {this.group = group;} else {this.group = stripDomain(group);}this.mode = mode;}@Overridepublic String toString() {return "Stat(owner='" + owner + "', group='" + group + "'" + ", mode=" + mode + ")";}public String getOwner() {return owner;}public String getGroup() {return group;}public int getMode() {return mode;}}/*** Returns the file stat for a file descriptor.** @param fd*            file descriptor.* @return the file descriptor file stat.* @throws IOException*             thrown if there was an IO error while obtaining the file stat.*/public static Stat getFstat(FileDescriptor fd) throws IOException {Stat stat = null;if (!Shell.WINDOWS) {stat = fstat(fd);stat.owner = getName(IdCache.USER, stat.ownerId);stat.group = getName(IdCache.GROUP, stat.groupId);} else {try {stat = fstat(fd);} catch (NativeIOException nioe) {if (nioe.getErrorCode() == 6) {throw new NativeIOException("The handle is invalid.", Errno.EBADF);} else {LOG.warn(String.format("NativeIO.getFstat error (%d): %s", nioe.getErrorCode(),nioe.getMessage()));throw new NativeIOException("Unknown error", Errno.UNKNOWN);}}}return stat;}private static String getName(IdCache domain, int id) throws IOException {Map<Integer, CachedName> idNameCache = (domain == IdCache.USER) ? USER_ID_NAME_CACHE : GROUP_ID_NAME_CACHE;String name;CachedName cachedName = idNameCache.get(id);long now = System.currentTimeMillis();if (cachedName != null && (cachedName.timestamp + cacheTimeout) > now) {name = cachedName.name;} else {name = (domain == IdCache.USER) ? getUserName(id) : getGroupName(id);if (LOG.isDebugEnabled()) {String type = (domain == IdCache.USER) ? "UserName" : "GroupName";LOG.debug("Got " + type + " " + name + " for ID " + id + " from the native implementation");}cachedName = new CachedName(name, now);idNameCache.put(id, cachedName);}return name;}static native String getUserName(int uid) throws IOException;static native String getGroupName(int uid) throws IOException;private static class CachedName {final long timestamp;final String name;public CachedName(String name, long timestamp) {this.name = name;this.timestamp = timestamp;}}private static final Map<Integer, CachedName> USER_ID_NAME_CACHE = new ConcurrentHashMap<Integer, CachedName>();private static final Map<Integer, CachedName> GROUP_ID_NAME_CACHE = new ConcurrentHashMap<Integer, CachedName>();private enum IdCache {USER, GROUP}public final static int MMAP_PROT_READ = 0x1;public final static int MMAP_PROT_WRITE = 0x2;public final static int MMAP_PROT_EXEC = 0x4;public static native long mmap(FileDescriptor fd, int prot, boolean shared, long length) throws IOException;public static native void munmap(long addr, long length) throws IOException;}private static boolean workaroundNonThreadSafePasswdCalls = false;public static class Windows {// Flags for CreateFile() call on Windowspublic static final long GENERIC_READ = 0x80000000L;public static final long GENERIC_WRITE = 0x40000000L;public static final long FILE_SHARE_READ = 0x00000001L;public static final long FILE_SHARE_WRITE = 0x00000002L;public static final long FILE_SHARE_DELETE = 0x00000004L;public static final long CREATE_NEW = 1;public static final long CREATE_ALWAYS = 2;public static final long OPEN_EXISTING = 3;public static final long OPEN_ALWAYS = 4;public static final long TRUNCATE_EXISTING = 5;public static final long FILE_BEGIN = 0;public static final long FILE_CURRENT = 1;public static final long FILE_END = 2;public static final long FILE_ATTRIBUTE_NORMAL = 0x00000080L;/*** Create a directory with permissions set to the specified mode. By setting* permissions at creation time, we avoid issues related to the user lacking* WRITE_DAC rights on subsequent chmod calls. One example where this can occur* is writing to an SMB share where the user does not have Full Control rights,* and therefore WRITE_DAC is denied.** @param path*            directory to create* @param mode*            permissions of new directory* @throws IOException*             if there is an I/O error*/public static void createDirectoryWithMode(File path, int mode) throws IOException {createDirectoryWithMode0(path.getAbsolutePath(), mode);}/** Wrapper around CreateDirectory() on Windows */private static native void createDirectoryWithMode0(String path, int mode) throws NativeIOException;/** Wrapper around CreateFile() on Windows */public static native FileDescriptor createFile(String path, long desiredAccess, long shareMode,long creationDisposition) throws IOException;/*** Create a file for write with permissions set to the specified mode. By* setting permissions at creation time, we avoid issues related to the user* lacking WRITE_DAC rights on subsequent chmod calls. One example where this* can occur is writing to an SMB share where the user does not have Full* Control rights, and therefore WRITE_DAC is denied.** This method mimics the semantics implemented by the JDK in* {@link FileOutputStream}. The file is opened for truncate or append,* the sharing mode allows other readers and writers, and paths longer than* MAX_PATH are supported. (See io_util_md.c in the JDK.)** @param path*            file to create* @param append*            if true, then open file for append* @param mode*            permissions of new directory* @return FileOutputStream of opened file* @throws IOException*             if there is an I/O error*/public static FileOutputStream createFileOutputStreamWithMode(File path, boolean append, int mode)throws IOException {long desiredAccess = GENERIC_WRITE;long shareMode = FILE_SHARE_READ | FILE_SHARE_WRITE;long creationDisposition = append ? OPEN_ALWAYS : CREATE_ALWAYS;return new FileOutputStream(createFileWithMode0(path.getAbsolutePath(), desiredAccess, shareMode, creationDisposition, mode));}/** Wrapper around CreateFile() with security descriptor on Windows */private static native FileDescriptor createFileWithMode0(String path, long desiredAccess, long shareMode,long creationDisposition, int mode) throws NativeIOException;/** Wrapper around SetFilePointer() on Windows */public static native long setFilePointer(FileDescriptor fd, long distanceToMove, long moveMethod)throws IOException;/** Windows only methods used for getOwner() implementation */private static native String getOwner(FileDescriptor fd) throws IOException;/** Supported list of Windows access right flags */public static enum AccessRight {ACCESS_READ(0x0001), // FILE_READ_DATAACCESS_WRITE(0x0002), // FILE_WRITE_DATAACCESS_EXECUTE(0x0020); // FILE_EXECUTEprivate final int accessRight;AccessRight(int access) {accessRight = access;}public int accessRight() {return accessRight;}};/*** Windows only method used to check if the current process has requested access* rights on the given path.*/private static native boolean access0(String path, int requestedAccess);/*** Checks whether the current process has desired access rights on the given* path.** Longer term this native function can be substituted with JDK7 function* Files#isReadable, isWritable, isExecutable.** @param path*            input path* @param desiredAccess*            ACCESS_READ, ACCESS_WRITE or ACCESS_EXECUTE* @return true if access is allowed* @throws IOException*             I/O exception on error*/public static boolean access(String path, AccessRight desiredAccess) throws IOException {return true;// return access0(path, desiredAccess.accessRight());}/*** Extends both the minimum and maximum working set size of the current process.* This method gets the current minimum and maximum working set size, adds the* requested amount to each and then sets the minimum and maximum working set* size to the new values. Controlling the working set size of the process also* controls the amount of memory it can lock.** @param delta*            amount to increment minimum and maximum working set size* @throws IOException*             for any error* @see POSIX#mlock(ByteBuffer, long)*/public static native void extendWorkingSetSize(long delta) throws IOException;static {if (NativeCodeLoader.isNativeCodeLoaded()) {try {initNative();nativeLoaded = true;} catch (Throwable t) {// This can happen if the user has an older version of libhadoop.so// installed - in this case we can continue without native IO// after warningPerformanceAdvisory.LOG.debug("Unable to initialize NativeIO libraries", t);}}}}private static final Log LOG = LogFactory.getLog(NativeIO.class);private static boolean nativeLoaded = false;static {if (NativeCodeLoader.isNativeCodeLoaded()) {try {initNative();nativeLoaded = true;} catch (Throwable t) {// This can happen if the user has an older version of libhadoop.so// installed - in this case we can continue without native IO// after warningPerformanceAdvisory.LOG.debug("Unable to initialize NativeIO libraries", t);}}}/*** Return true if the JNI-based native IO extensions are available.*/public static boolean isAvailable() {return NativeCodeLoader.isNativeCodeLoaded() && nativeLoaded;}/** Initialize the JNI method ID and class ID cache */private static native void initNative();/*** Get the maximum number of bytes that can be locked into memory at any given* point.** @return 0 if no bytes can be locked into memory; Long.MAX_VALUE if there is*         no limit; The number of bytes that can be locked into memory*         otherwise.*/static long getMemlockLimit() {return isAvailable() ? getMemlockLimit0() : 0;}private static native long getMemlockLimit0();/*** @return the operating system's page size.*/static long getOperatingSystemPageSize() {try {Field f = Unsafe.class.getDeclaredField("theUnsafe");f.setAccessible(true);Unsafe unsafe = (Unsafe) f.get(null);return unsafe.pageSize();} catch (Throwable e) {LOG.warn("Unable to get operating system page size.  Guessing 4096.", e);return 4096;}}private static class CachedUid {final long timestamp;final String username;public CachedUid(String username, long timestamp) {this.timestamp = timestamp;this.username = username;}}private static final Map<Long, CachedUid> uidCache = new ConcurrentHashMap<Long, CachedUid>();private static long cacheTimeout;private static boolean initialized = false;/*** The Windows logon name has two part, NetBIOS domain name and user account* name, of the format DOMAIN\UserName. This method will remove the domain part* of the full logon name.** @param Fthe*            full principal name containing the domain* @return name with domain removed*/private static String stripDomain(String name) {int i = name.indexOf('\\');if (i != -1)name = name.substring(i + 1);return name;}public static String getOwner(FileDescriptor fd) throws IOException {ensureInitialized();if (Shell.WINDOWS) {String owner = Windows.getOwner(fd);owner = stripDomain(owner);return owner;} else {long uid = POSIX.getUIDforFDOwnerforOwner(fd);CachedUid cUid = uidCache.get(uid);long now = System.currentTimeMillis();if (cUid != null && (cUid.timestamp + cacheTimeout) > now) {return cUid.username;}String user = POSIX.getUserName(uid);LOG.info("Got UserName " + user + " for UID " + uid + " from the native implementation");cUid = new CachedUid(user, now);uidCache.put(uid, cUid);return user;}}/*** Create a FileInputStream that shares delete permission on the file opened,* i.e. other process can delete the file the FileInputStream is reading. Only* Windows implementation uses the native interface.*/public static FileInputStream getShareDeleteFileInputStream(File f) throws IOException {if (!Shell.WINDOWS) {// On Linux the default FileInputStream shares delete permission// on the file opened.//return new FileInputStream(f);} else {// Use Windows native interface to create a FileInputStream that// shares delete permission on the file opened.//FileDescriptor fd = Windows.createFile(f.getAbsolutePath(), Windows.GENERIC_READ,Windows.FILE_SHARE_READ | Windows.FILE_SHARE_WRITE | Windows.FILE_SHARE_DELETE,Windows.OPEN_EXISTING);return new FileInputStream(fd);}}/*** Create a FileInputStream that shares delete permission on the file opened at* a given offset, i.e. other process can delete the file the FileInputStream is* reading. Only Windows implementation uses the native interface.*/public static FileInputStream getShareDeleteFileInputStream(File f, long seekOffset) throws IOException {if (!Shell.WINDOWS) {RandomAccessFile rf = new RandomAccessFile(f, "r");if (seekOffset > 0) {rf.seek(seekOffset);}return new FileInputStream(rf.getFD());} else {// Use Windows native interface to create a FileInputStream that// shares delete permission on the file opened, and set it to the// given offset.//FileDescriptor fd = NativeIO.Windows.createFile(f.getAbsolutePath(), NativeIO.Windows.GENERIC_READ, NativeIO.Windows.FILE_SHARE_READ| NativeIO.Windows.FILE_SHARE_WRITE | NativeIO.Windows.FILE_SHARE_DELETE,NativeIO.Windows.OPEN_EXISTING);if (seekOffset > 0)NativeIO.Windows.setFilePointer(fd, seekOffset, NativeIO.Windows.FILE_BEGIN);return new FileInputStream(fd);}}/*** Create the specified File for write access, ensuring that it does not exist.** @param f*            the file that we want to create* @param permissions*            we want to have on the file (if security is enabled)** @throws AlreadyExistsException*             if the file already exists* @throws IOException*             if any other error occurred*/public static FileOutputStream getCreateForWriteFileOutputStream(File f, int permissions) throws IOException {if (!Shell.WINDOWS) {// Use the native wrapper around open(2)try {FileDescriptor fd = NativeIO.POSIX.open(f.getAbsolutePath(),NativeIO.POSIX.O_WRONLY | NativeIO.POSIX.O_CREAT | NativeIO.POSIX.O_EXCL, permissions);return new FileOutputStream(fd);} catch (NativeIOException nioe) {if (nioe.getErrno() == Errno.EEXIST) {throw new AlreadyExistsException(nioe);}throw nioe;}} else {// Use the Windows native APIs to create equivalent FileOutputStreamtry {FileDescriptor fd = NativeIO.Windows.createFile(f.getCanonicalPath(), NativeIO.Windows.GENERIC_WRITE, NativeIO.Windows.FILE_SHARE_DELETE| NativeIO.Windows.FILE_SHARE_READ | NativeIO.Windows.FILE_SHARE_WRITE,NativeIO.Windows.CREATE_NEW);NativeIO.POSIX.chmod(f.getCanonicalPath(), permissions);return new FileOutputStream(fd);} catch (NativeIOException nioe) {if (nioe.getErrorCode() == 80) {// ERROR_FILE_EXISTS// 80 (0x50)// The file existsthrow new AlreadyExistsException(nioe);}throw nioe;}}}private synchronized static void ensureInitialized() {if (!initialized) {cacheTimeout = new Configuration().getLong("hadoop.security.uid.cache.secs", 4 * 60 * 60) * 1000;LOG.info("Initialized cache for UID to User mapping with a cache" + " timeout of " + cacheTimeout / 1000+ " seconds.");initialized = true;}}/*** A version of renameTo that throws a descriptive exception when it fails.** @param src*            The source path* @param dst*            The destination path** @throws NativeIOException*             On failure.*/public static void renameTo(File src, File dst) throws IOException {if (!nativeLoaded) {if (!src.renameTo(dst)) {throw new IOException("renameTo(src=" + src + ", dst=" + dst + ") failed.");}} else {renameTo0(src.getAbsolutePath(), dst.getAbsolutePath());}}public static void link(File src, File dst) throws IOException {if (!nativeLoaded) {HardLink.createHardLink(src, dst);} else {link0(src.getAbsolutePath(), dst.getAbsolutePath());}}/*** A version of renameTo that throws a descriptive exception when it fails.** @param src*            The source path* @param dst*            The destination path** @throws NativeIOException*             On failure.*/private static native void renameTo0(String src, String dst) throws NativeIOException;private static native void link0(String src, String dst) throws NativeIOException;/*** Unbuffered file copy from src to dst without tainting OS buffer cache** In POSIX platform: It uses FileChannel#transferTo() which internally attempts* unbuffered IO on OS with native sendfile64() support and falls back to* buffered IO otherwise.** It minimizes the number of FileChannel#transferTo call by passing the the src* file size directly instead of a smaller size as the 3rd parameter. This saves* the number of sendfile64() system call when native sendfile64() is supported.* In the two fall back cases where sendfile is not supported,* FileChannle#transferTo already has its own batching of size 8 MB and 8 KB,* respectively.** In Windows Platform: It uses its own native wrapper of CopyFileEx with* COPY_FILE_NO_BUFFERING flag, which is supported on Windows Server 2008 and* above.** Ideally, we should use FileChannel#transferTo() across both POSIX and Windows* platform. Unfortunately, the* wrapper(Java_sun_nio_ch_FileChannelImpl_transferTo0) used by* FileChannel#transferTo for unbuffered IO is not implemented on Windows. Based* on OpenJDK 6/7/8 source code, Java_sun_nio_ch_FileChannelImpl_transferTo0 on* Windows simply returns IOS_UNSUPPORTED.** Note: This simple native wrapper does minimal parameter checking before copy* and consistency check (e.g., size) after copy. It is recommended to use* wrapper function like the Storage#nativeCopyFileUnbuffered() function in* hadoop-hdfs with pre/post copy checks.** @param src*            The source path* @param dst*            The destination path* @throws IOException*/public static void copyFileUnbuffered(File src, File dst) throws IOException {if (nativeLoaded && Shell.WINDOWS) {copyFileUnbuffered0(src.getAbsolutePath(), dst.getAbsolutePath());} else {FileInputStream fis = null;FileOutputStream fos = null;FileChannel input = null;FileChannel output = null;try {fis = new FileInputStream(src);fos = new FileOutputStream(dst);input = fis.getChannel();output = fos.getChannel();long remaining = input.size();long position = 0;long transferred = 0;while (remaining > 0) {transferred = input.transferTo(position, remaining, output);remaining -= transferred;position += transferred;}} finally {IOUtils.cleanup(LOG, output);IOUtils.cleanup(LOG, fos);IOUtils.cleanup(LOG, input);IOUtils.cleanup(LOG, fis);}}}private static native void copyFileUnbuffered0(String src, String dst) throws NativeIOException;
}

本文来自互联网用户投稿，该文观点仅代表作者本人，不代表本站立场。本站仅提供信息存储空间服务，不拥有所有权，不承担相关法律责任。如若转载，请注明出处：http://www.mzph.cn/news/31709.shtml

如若内容造成侵权/违法违规/事实不符，请联系多彩编程网进行投诉反馈email:809451989@qq.com，一经查实，立即删除！

相关文章

黑马程序员Vue全套视频

黑马程序员Vue全套视频

Vue 2 创建vue实例 示例: <head><meta charset"UTF-8"><meta name"viewport" content"widthdevice-width, initial-scale1.0"><title>Document</title><script src"https://cdn.jsdelivr.net/npm/vue2.…
阅读更多...
液体神经网络：LNN是个啥概念？

液体神经网络：LNN是个啥概念？

一、说明 在在人工智能领域&#xff0c;神经网络已被证明是解决复杂问题的非常强大的工具。多年来&#xff0c;研究人员不断寻求创新方法来提高其性能并扩展其能力。其中一种方法是液体神经网络&#xff08;LNN&#xff09;的概念&#xff0c;这是一个利用动态计算功能的迷人框…
阅读更多...
Field Symbol与数据引用的几个应用

Field Symbol与数据引用的几个应用

这俩货都是指针。在动态编程里用。 但是从好理解的角度来看&#xff0c;都给他们理解成数据对象。 都得指向其他的数据对象。不过field symbol指的是其他的数据对象内存地址的值。而数据引用只是指向内存地址。 1.Field symbol 当Field Symbol指向了其他的数据对象&#xf…
阅读更多...
【实操干货】如何开始用Qt Widgets编程？（二）

【实操干货】如何开始用Qt Widgets编程？（二）

Qt 是目前最先进、最完整的跨平台C开发工具。它不仅完全实现了一次编写&#xff0c;所有平台无差别运行&#xff0c;更提供了几乎所有开发过程中需要用到的工具。如今&#xff0c;Qt已被运用于超过70个行业、数千家企业&#xff0c;支持数百万设备及应用。 在本文中&#xff0…
阅读更多...
解决: git拉取报错 git 未能顺利结束 (退出码 1)

解决: git拉取报错 git 未能顺利结束 (退出码 1)

拉取代码失败信息 解决方法: 执行一下"git push -f origin master"命令即可 步骤: 1.项目文件夹右击选择"Git Bash Here",打开命令窗口 2. 输入"git push -f origin master"后,回画 执行结束 3.再拉取代码,成功
阅读更多...
pytest结合 allure 打标记之的详细使用

pytest结合 allure 打标记之的详细使用

前言 前面我们提到使用allure 可以生成漂亮的测试报告&#xff0c;下面就Allure 标记我们做详细介绍。 allure 标记 包含&#xff1a;epic&#xff0c;feature, story, title, testcase, issue, description, step, serverity, link, attachment 常用的标记 allure.feature…
阅读更多...
20230807通过ffmpeg将DTS编码的AUDIO音频转换为AAC编码

20230807通过ffmpeg将DTS编码的AUDIO音频转换为AAC编码

20230807通过ffmpeg将DTS编码的AUDIO音频转换为AAC编码 2023/8/7 20:04 ffmpeg dts 转AAC 缘起&#xff1a;由于网上找的电影没有中文字幕&#xff0c;有内置的英文字幕&#xff0c;但是还是通过剪映/RP2023识别一份英文字幕备用&#xff01; I:\Downloads\2005[红眼航班]Red E…
阅读更多...
数据结构 | 利用二叉堆实现优先级队列

数据结构 | 利用二叉堆实现优先级队列

目录 一、二叉堆的操作 二、二叉堆的实现 2.1 结构属性 2.2 堆的有序性 2.3 堆操作 队列有一个重要的变体&#xff0c;叫作优先级队列。和队列一样&#xff0c;优先级队列从头部移除元素&#xff0c;不过元素的逻辑顺序是由优先级决定的。优先级最高的元素在最前&#xff…
阅读更多...
PHP最简单自定义自己的框架控制器自动加载运行（四）

PHP最简单自定义自己的框架控制器自动加载运行（四）

1、实现效果调用控制中方法 2、创建控制器indexCrl.php <?php class indexCrl{public function index(){echo 当前index控制器index方法;} } 3、KJ.php字段加载控制器文件 public static function run(){//定义常量self::_set_const();//创建模块目录self::_mk_module();…
阅读更多...
finfet grid

finfet grid

我正在「拾陆楼」和朋友们讨论有趣的话题&#xff0c;你⼀起来吧&#xff1f; 拾陆楼知识星球入口 90nm 及以下的工艺都要求储存器&#xff0c;IP&#xff0c;IO 的多晶硅方向必须和标准单元的多晶 硅方向保持一致&#xff0c;无法像过去工艺一样随意旋转方向。在 22nm 及以下…
阅读更多...
C++初阶语法——类型指示符auto和空指针nullptr

C++初阶语法——类型指示符auto和空指针nullptr

前言&#xff1a;本篇文章的重点 1.C中的类型指示符auto 2.借由auto实现的范围for 3.C为什么要创建nullptr而不是用原本的NULL。 目录 一.auto1.auto的含义及使用2.auto使用注意事项3.范围for&#xff08;重头戏&#xff09;范围for注意事项 二.nullptr 一.auto 1.auto的含义及…
阅读更多...
【css】添加待小三角的提示框

【css】添加待小三角的提示框

如需创建在工具提示的指定侧面显示的箭头&#xff0c;在工具提示后添加“空的”内容&#xff0c;并使用伪元素类 ::after 和 content 属性。箭头本身是使用边框创建的。这会使工具提示看起来像气泡。 代码&#xff1a; <style> .tooltip {position: relative;display: …
阅读更多...
【选择排序】直接选择排序 与 堆排序

【选择排序】直接选择排序 与 堆排序

目录 1. 排序的概念&#xff1a; 2.选择排序的基本思想 3.直接选择排序 4.堆排序 1. 排序的概念&#xff1a; 排序&#xff1a;所谓排序&#xff0c;就是使一串记录&#xff0c;按照其中的某个或某些关键字的大小&#xff0c;递增或递减的排列起来的操作。 稳定性&#xf…
阅读更多...
STM32 F103C8T6学习笔记4：时钟树、滴答计时器、定时器定时中断

STM32 F103C8T6学习笔记4：时钟树、滴答计时器、定时器定时中断

今日理解一下STM32F103 C8T6的时钟与时钟系统、滴答计时器、定时器计时中断的配置&#xff0c;文章提供原理&#xff0c;代码&#xff0c;测试工程下载。 目录 时钟树与时钟系统&#xff1a; 滴答计时器&#xff1a; 定时器计时中断&#xff1a; 测试结果&#xff1a; 测…
阅读更多...
机器学习、深度学习项目开发业务数据场景梳理汇总记录五

机器学习、深度学习项目开发业务数据场景梳理汇总记录五

本文的主要作用是对历史项目开发过程中接触到的业务数据进行整体的汇总梳理&#xff0c;文章会随着项目的开发推进不断更新。 这里是续文&#xff0c;因为CSDN单篇文章内容太大的话就会崩溃的&#xff0c;别问我怎么知道的&#xff0c;问就是血泪教训&#xff0c;辛辛苦苦写了一…
阅读更多...
Vue [Day6]

Vue [Day6]

路由进阶 路由模块的封装抽离 src/router/index.js import VueRouter from vue-router // 用绝对路径的方式来写目录 相当于src import Find from /views/Find import Friend from ../views/Friend import My from ../views/Myimport Vue from vue Vue.use(VueRouter)con…
阅读更多...
2023年加湿器市场数据分析（天猫数据分析怎么做）

2023年加湿器市场数据分析（天猫数据分析怎么做）

随着人们生活水平提高、空调广泛使用&#xff0c;导致皮肤紧绷、口舌干燥、咳嗽感冒等空调病的滋生。可以看出&#xff0c;空气湿度与人体健康以及日常生活有着密切的联系。而加湿器作为室内空气湿度控制的重要工具&#xff0c;在近年来受到了越来越多的重视。根据鲸参谋电商数…
阅读更多...
Godot 4 源码分析 - 文件读入编码处理

Godot 4 源码分析 - 文件读入编码处理

今天需要读入xml文件进行处理&#xff0c;结果读入一个带中文的文件时&#xff0c;出错了。当然程序还能运行&#xff0c;但编译器一直报错&#xff0c;而且XML解析也不正确 单步调试发现读入的内容出现乱码&#xff0c;具体逻辑&#xff1a; String FileAccess::get_as_text…
阅读更多...
物联网与5G引领智慧城市时代的到来

物联网与5G引领智慧城市时代的到来

智慧城市需要依赖于多种技术&#xff0c;这些技术的应用将城市转变为高效、智能、可持续发展的现代化城市。智慧城市是基于信息技术、物联网和大数据等先进技术的融合&#xff0c;旨在提升城市的运行效率、资源利用效率和居民生活质量。以下是智慧城市需要依赖的主要技术&#…
阅读更多...
Kafka3.4 SASL/kerberos/ACL 证以及 SSL 加密连接

Kafka3.4 SASL/kerberos/ACL 证以及 SSL 加密连接

Kafka3.4 SASL/kerberos ACL 证以及 SSL 加密连接 序 前面我们使用 kafka3.3.1 on zookeeper 的模式进行多网段监听的 kafka 集群&#xff0c;顺便搭建起 kafkaui 后发现一些问题&#xff0c;我们 kafka 集群没有连接认证&#xff0c;万一谁知道了我们的 kafka 连接地址&…
阅读更多...
推荐文章
最新文章

Copyright @ 2022~2023