Skip to content

Commit

Permalink
HDFS-10756. Expose getTrashRoot to HTTPFS and WebHDFS. Contributed by…
Browse files Browse the repository at this point in the history 
… Yuanbo Liu.
  • Loading branch information
@xiao-chen
xiao-chen committed Nov 5, 2016
1 parent 95665a6 commit d8bab3d
Show file tree
Hide file tree
Showing 15 changed files with 366 additions and 88 deletions.
3 changes: 2 additions & 1 deletion hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -113,6 +113,7 @@ public abstract class FileSystem extends Configured implements Closeable {
public static final int SHUTDOWN_HOOK_PRIORITY = 10;

public static final String TRASH_PREFIX = ".Trash";
public static final String USER_HOME_PREFIX = "/user";

/** FileSystem cache */
static final Cache CACHE = new Cache();
Expand Down Expand Up @@ -1965,7 +1966,7 @@ public LocatedFileStatus next() throws IOException {
*/
public Path getHomeDirectory() {
return this.makeQualified(
new Path("/user/"+System.getProperty("user.name")));
new Path(USER_HOME_PREFIX + "/" + System.getProperty("user.name")));
}


Expand Down
1 change: 1 addition & 0 deletions ...roject/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOpsCountStatistics.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,7 @@ public enum OpType {
GET_STATUS(CommonStatisticNames.OP_GET_STATUS),
GET_STORAGE_POLICIES("op_get_storage_policies"),
GET_STORAGE_POLICY("op_get_storage_policy"),
GET_TRASH_ROOT("op_get_trash_root"),
GET_XATTR("op_get_xattr"),
LIST_LOCATED_STATUS(CommonStatisticNames.OP_LIST_LOCATED_STATUS),
LIST_STATUS(CommonStatisticNames.OP_LIST_STATUS),
Expand Down
21 changes: 21 additions & 0 deletions ...roject/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1621,6 +1621,27 @@ BlockLocation[] decodeResponse(Map<?,?> json) throws IOException {
}.run();
}

@Override
public Path getTrashRoot(Path path) {
statistics.incrementReadOps(1);
storageStatistics.incrementOpCounter(OpType.GET_TRASH_ROOT);

final HttpOpParam.Op op = GetOpParam.Op.GETTRASHROOT;
try {
String strTrashPath = new FsPathResponseRunner<String>(op, path) {
@Override
String decodeResponse(Map<?, ?> json) throws IOException {
return JsonUtilClient.getPath(json);
}
}.run();
return new Path(strTrashPath).makeQualified(getUri(), null);
} catch(IOException e) {
LOG.warn("Cannot find trash root of " + path, e);
// keep the same behavior with dfs
return super.getTrashRoot(path).makeQualified(getUri(), null);
}
}

@Override
public void access(final Path path, final FsAction mode) throws IOException {
final HttpOpParam.Op op = GetOpParam.Op.CHECKACCESS;
Expand Down
1 change: 1 addition & 0 deletions ...ect/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/GetOpParam.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@ public enum Op implements HttpOpParam.Op {
GET_BLOCK_LOCATIONS(false, HttpURLConnection.HTTP_OK),
GETACLSTATUS(false, HttpURLConnection.HTTP_OK),
GETXATTRS(false, HttpURLConnection.HTTP_OK),
GETTRASHROOT(false, HttpURLConnection.HTTP_OK),
LISTXATTRS(false, HttpURLConnection.HTTP_OK),

NULL(false, HttpURLConnection.HTTP_NOT_IMPLEMENTED),
Expand Down
31 changes: 30 additions & 1 deletion ...t/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -128,6 +128,8 @@ public class HttpFSFileSystem extends FileSystem

public static final String HOME_DIR_JSON = "Path";

public static final String TRASH_DIR_JSON = "Path";

public static final String SET_REPLICATION_JSON = "boolean";

public static final String UPLOAD_CONTENT_TYPE= "application/octet-stream";
Expand Down Expand Up @@ -203,7 +205,7 @@ public static enum Operation {
OPEN(HTTP_GET), GETFILESTATUS(HTTP_GET), LISTSTATUS(HTTP_GET),
GETHOMEDIRECTORY(HTTP_GET), GETCONTENTSUMMARY(HTTP_GET),
GETFILECHECKSUM(HTTP_GET), GETFILEBLOCKLOCATIONS(HTTP_GET),
INSTRUMENTATION(HTTP_GET), GETACLSTATUS(HTTP_GET),
INSTRUMENTATION(HTTP_GET), GETACLSTATUS(HTTP_GET), GETTRASHROOT(HTTP_GET),
APPEND(HTTP_POST), CONCAT(HTTP_POST), TRUNCATE(HTTP_POST),
CREATE(HTTP_PUT), MKDIRS(HTTP_PUT), RENAME(HTTP_PUT), SETOWNER(HTTP_PUT),
SETPERMISSION(HTTP_PUT), SETREPLICATION(HTTP_PUT), SETTIMES(HTTP_PUT),
Expand Down Expand Up @@ -818,6 +820,33 @@ public Path getHomeDirectory() {
}
}

/**
* Get the root directory of Trash for a path in HDFS.
* 1. File in encryption zone returns /ez1/.Trash/username.
* 2. File not in encryption zone, or encountered exception when checking
* the encryption zone of the path, returns /users/username/.Trash.
* Caller appends either Current or checkpoint timestamp
* for trash destination.
* The default implementation returns "/user/username/.Trash".
* @param fullPath the trash root of the path to be determined.
* @return trash root
*/
@Override
public Path getTrashRoot(Path fullPath) {
Map<String, String> params = new HashMap<>();
params.put(OP_PARAM, Operation.GETTRASHROOT.toString());
try {
HttpURLConnection conn = getConnection(
Operation.GETTRASHROOT.getMethod(), params, fullPath, true);
HttpExceptionUtils.validateResponse(conn, HttpURLConnection.HTTP_OK);
JSONObject json = (JSONObject) HttpFSUtils.jsonParse(conn);
return new Path((String) json.get(TRASH_DIR_JSON));
} catch (IOException ex) {
LOG.warn("Cannot find trash root of " + fullPath, ex);
return super.getTrashRoot(fullPath);
}
}

/**
* Set owner of a path (i.e. a file or a directory).
* The parameters username and groupname cannot both be null.
Expand Down
23 changes: 23 additions & 0 deletions ...oject/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/FSOperations.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1056,6 +1056,29 @@ public Void execute(FileSystem fs) throws IOException {

}

/**
* Executor that performs getting trash root FileSystemAccess
* files system operation.
*/
@InterfaceAudience.Private
public static class FSTrashRoot
implements FileSystemAccess.FileSystemExecutor<JSONObject> {
private Path path;
public FSTrashRoot(String path) {
this.path = new Path(path);
}

@Override
@SuppressWarnings("unchecked")
public JSONObject execute(FileSystem fs) throws IOException {
Path trashRoot = fs.getTrashRoot(this.path);
JSONObject json = new JSONObject();
json.put(HttpFSFileSystem.TRASH_DIR_JSON, trashRoot.toUri().getPath());
return json;
}

}

/**
* Executor that gets the ACL information for a given file.
*/
Expand Down
1 change: 1 addition & 0 deletions ...-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSParametersProvider.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,7 @@ public class HttpFSParametersProvider extends ParametersProvider {
PARAMS_DEF.put(Operation.GETFILECHECKSUM, new Class[]{});
PARAMS_DEF.put(Operation.GETFILEBLOCKLOCATIONS, new Class[]{});
PARAMS_DEF.put(Operation.GETACLSTATUS, new Class[]{});
PARAMS_DEF.put(Operation.GETTRASHROOT, new Class[]{});
PARAMS_DEF.put(Operation.INSTRUMENTATION, new Class[]{});
PARAMS_DEF.put(Operation.APPEND, new Class[]{DataParam.class});
PARAMS_DEF.put(Operation.CONCAT, new Class[]{SourcesParam.class});
Expand Down
7 changes: 7 additions & 0 deletions ...oject/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSServer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -339,6 +339,13 @@ public InputStream run() throws Exception {
response = Response.ok(json).type(MediaType.APPLICATION_JSON).build();
break;
}
case GETTRASHROOT: {
FSOperations.FSTrashRoot command = new FSOperations.FSTrashRoot(path);
JSONObject json = fsExecute(user, command);
AUDIT_LOG.info("[{}]", path);
response = Response.ok(json).type(MediaType.APPLICATION_JSON).build();
break;
}
default: {
throw new IOException(
MessageFormat.format("Invalid HTTP GET operation [{0}]", op.value()));
Expand Down
2 changes: 2 additions & 0 deletions hadoop-hdfs-project/hadoop-hdfs-httpfs/src/site/markdown/index.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,8 @@ HttpFS HTTP web-service API calls are HTTP REST calls that map to a HDFS file sy

* `$ curl http://httpfs-host:14000/webhdfs/v1/user/foo?op=list` returns the contents of the HDFS `/user/foo` directory in JSON format.

* `$ curl http://httpfs-host:14000/webhdfs/v1/user/foo?op=GETTRASHROOT` returns the path `/user/foo/.Trash`, if `/` is an encrypted zone, returns the path `/.Trash/foo`. See [more details](../hadoop-project-dist/hadoop-hdfs/TransparentEncryption.html#Rename_and_Trash_considerations) about trash path in an encrypted zone.

* `$ curl -X POST http://httpfs-host:14000/webhdfs/v1/user/foo/bar?op=mkdirs` creates the HDFS `/user/foo.bar` directory.

User and Developer Documentation
Expand Down
185 changes: 109 additions & 76 deletions ...hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/BaseTestHttpFSWith.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -430,6 +430,35 @@ private void testWorkingdirectory() throws Exception {
new Path("/tmp").toUri().getPath());
}

private void testTrashRoot() throws Exception {
if (!isLocalFS()) {
FileSystem fs = FileSystem.get(getProxiedFSConf());

final Path rootDir = new Path("/");
final Path fooPath = new Path(getProxiedFSTestDir(), "foo.txt");
OutputStream os = fs.create(fooPath);
os.write(1);
os.close();

Path trashPath = fs.getTrashRoot(rootDir);
Path fooTrashPath = fs.getTrashRoot(fooPath);
fs.close();

fs = getHttpFSFileSystem();
Path httpFSTrashPath = fs.getTrashRoot(rootDir);
Path httpFSFooTrashPath = fs.getTrashRoot(fooPath);
fs.close();

assertEquals(trashPath.toUri().getPath(),
httpFSTrashPath.toUri().getPath());
assertEquals(fooTrashPath.toUri().getPath(),
httpFSFooTrashPath.toUri().getPath());
// trash path is related to USER, not path
assertEquals(trashPath.toUri().getPath(),
fooTrashPath.toUri().getPath());
}
}

private void testMkdirs() throws Exception {
Path path = new Path(getProxiedFSTestDir(), "foo");
FileSystem fs = getHttpFSFileSystem();
Expand Down Expand Up @@ -916,86 +945,90 @@ protected enum Operation {
GET, OPEN, CREATE, APPEND, TRUNCATE, CONCAT, RENAME, DELETE, LIST_STATUS,
WORKING_DIRECTORY, MKDIRS, SET_TIMES, SET_PERMISSION, SET_OWNER,
SET_REPLICATION, CHECKSUM, CONTENT_SUMMARY, FILEACLS, DIRACLS, SET_XATTR,
GET_XATTRS, REMOVE_XATTR, LIST_XATTRS, ENCRYPTION, LIST_STATUS_BATCH
GET_XATTRS, REMOVE_XATTR, LIST_XATTRS, ENCRYPTION, LIST_STATUS_BATCH,
GETTRASHROOT
}

private void operation(Operation op) throws Exception {
switch (op) {
case GET:
testGet();
break;
case OPEN:
testOpen();
break;
case CREATE:
testCreate();
break;
case APPEND:
testAppend();
break;
case TRUNCATE:
testTruncate();
break;
case CONCAT:
testConcat();
break;
case RENAME:
testRename();
break;
case DELETE:
testDelete();
break;
case LIST_STATUS:
testListStatus();
break;
case WORKING_DIRECTORY:
testWorkingdirectory();
break;
case MKDIRS:
testMkdirs();
break;
case SET_TIMES:
testSetTimes();
break;
case SET_PERMISSION:
testSetPermission();
break;
case SET_OWNER:
testSetOwner();
break;
case SET_REPLICATION:
testSetReplication();
break;
case CHECKSUM:
testChecksum();
break;
case CONTENT_SUMMARY:
testContentSummary();
break;
case FILEACLS:
testFileAcls();
break;
case DIRACLS:
testDirAcls();
break;
case SET_XATTR:
testSetXAttr();
break;
case REMOVE_XATTR:
testRemoveXAttr();
break;
case GET_XATTRS:
testGetXAttrs();
break;
case LIST_XATTRS:
testListXAttrs();
break;
case ENCRYPTION:
testEncryption();
break;
case LIST_STATUS_BATCH:
testListStatusBatch();
break;
case GET:
testGet();
break;
case OPEN:
testOpen();
break;
case CREATE:
testCreate();
break;
case APPEND:
testAppend();
break;
case TRUNCATE:
testTruncate();
break;
case CONCAT:
testConcat();
break;
case RENAME:
testRename();
break;
case DELETE:
testDelete();
break;
case LIST_STATUS:
testListStatus();
break;
case WORKING_DIRECTORY:
testWorkingdirectory();
break;
case MKDIRS:
testMkdirs();
break;
case SET_TIMES:
testSetTimes();
break;
case SET_PERMISSION:
testSetPermission();
break;
case SET_OWNER:
testSetOwner();
break;
case SET_REPLICATION:
testSetReplication();
break;
case CHECKSUM:
testChecksum();
break;
case CONTENT_SUMMARY:
testContentSummary();
break;
case FILEACLS:
testFileAcls();
break;
case DIRACLS:
testDirAcls();
break;
case SET_XATTR:
testSetXAttr();
break;
case REMOVE_XATTR:
testRemoveXAttr();
break;
case GET_XATTRS:
testGetXAttrs();
break;
case LIST_XATTRS:
testListXAttrs();
break;
case ENCRYPTION:
testEncryption();
break;
case LIST_STATUS_BATCH:
testListStatusBatch();
break;
case GETTRASHROOT:
testTrashRoot();
break;
}
}

Expand Down
Loading

0 comments on commit d8bab3d

Please sign in to comment.