amer/jabba
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

etag support for file server added

Browse Source
This commit is contained in:
Amer Agovic
2024-01-16 23:30:25 -06:00
parent ba9b980fea
commit 5e77c270d9
12 changed files with 359 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files
extra.gradle
+1
View File
@@ -115,6 +115,7 @@ task runServer{
}
doLast {
Server.main().start({
println(application.mainClass.get())
project.javaexec {
classpath = sourceSets.main.runtimeClasspath
main = application.mainClass.get()
src/main/java/com/reliancy/jabba/ArgsConfig.java
+18 -8
View File
@@ -86,18 +86,15 @@ public class ArgsConfig extends Config.Base{
}else if(values.size()==1){
// we got single value
String val=values.get(0);
Property<?> prop=null;
if("true".equalsIgnoreCase(val) || "false".equalsIgnoreCase(val)){
Property<Boolean> prop=new Property<>(pkey,Boolean.class);
Boolean val2=prop.adaptValue(val);
setProperty(prop,val2);
prop=new Property<>(pkey,Boolean.class);
}else if(Handy.isNumeric(val)){
Property<Float> prop=new Property<>(pkey,Float.class);
Float val2=prop.adaptValue(val);
setProperty(prop,val2);
prop=new Property<>(pkey,Float.class);
}else{
Property<String> prop=new Property<>(pkey,String.class);
setProperty(prop,val);
prop=new Property<>(pkey,String.class);
}
prop.setString(this, val);
}else{
// we got a list
Property<List> prop=new Property<>(pkey,List.class);
@@ -119,6 +116,19 @@ public class ArgsConfig extends Config.Base{
if(!cwd.endsWith("/")) cwd+="/";
APP_WORKDIR.set(this, cwd);
}
String conf=null;
String[] confs=new String[]{APP_SETTINGS.get(this),"./etc","./conf","./config","../etc","../conf","../config"};
for(String c:confs){
if(c==null) continue;
File f=new File(c);
if(f.exists() && f.isFile()){
conf=c;break;
}
}
if(conf!=null){ // we got settings file
conf=conf.replace("\\", "/").replace("/./","/");
APP_SETTINGS.set(this, cwd);
}
// also logging level and format
Logger root=Log.setup();
Log.setLevel(root,LOG_LEVEL.get(this));
src/main/java/com/reliancy/jabba/Config.java
+20 -2
View File
@@ -25,11 +25,13 @@ public interface Config extends Iterable<Config.Property<?>>{
V initial;
boolean required;
boolean writable;
boolean persistent;
public Property(String name,Class<V> typ){
this.name=name;
this.typ=typ;
required=false;
writable=true;
persistent=false;
}
@Override
public String toString(){
@@ -43,14 +45,24 @@ public interface Config extends Iterable<Config.Property<?>>{
public boolean isRequired(){return required;}
public Property<V> setWritable(boolean f){writable=f;return this;}
public boolean isWritable(){return writable;}
public Property<V> setPersistent(boolean f){persistent=f;return this;}
public boolean isPersistent(){return persistent;}
public V get(Config store,V def){
return store.getProperty(this,def);
}
public V get(Config store){
return get(store,initial);
}
public void set(Config store,V val){
public String getString(Config conf){
V val=get(conf);
return String.valueOf(Handy.normalize(String.class,val));
}
public Property<V> set(Config store,V val){
store.setProperty(this, val);
return this;
}
public void setString(Config store,String val){
store.setProperty(this, adaptValue(val));
}
/** converts value such as string to expected type if possible. */
public V adaptValue(Object val){
@@ -100,6 +112,7 @@ public interface Config extends Iterable<Config.Property<?>>{
}
@Override
public <T> Config setProperty(Property<T> key, T val) {
//if(!key.isWritable()) throw new RuntimeException("read only property:"+key);
setModified(key);
props.put(key,val);
return this;
@@ -121,6 +134,10 @@ public interface Config extends Iterable<Config.Property<?>>{
for(Property<?> pp:p) schema.add(pp);
return this;
}
@Override
public Property<?>[] getSchema(){
return schema.toArray(new Property<?>[schema.size()]);
}
}
public static final Property<String> LOG_LEVEL=new Property<>("LOG_LEVEL",String.class);
@@ -130,6 +147,7 @@ public interface Config extends Iterable<Config.Property<?>>{
public static final Property<String> APP_TITLE=new Property<>("APP_TITLE",String.class);
public static final Property<String> APP_INFO=new Property<>("APP_INFO",String.class);
public static final Property<String> APP_WORKDIR=new Property<>("APP_WORKDIR",String.class);
public static final Property<String> APP_SETTINGS=new Property<>("APP_SETTINGS",String.class);
public static final Property<String> APP_CLASS=new Property<>("APP_CLASS",String.class);
public static final Property<List> APP_ARGS=new Property<>("APP_ARGS",List.class);
@@ -143,5 +161,5 @@ public interface Config extends Iterable<Config.Property<?>>{
public <T> T getProperty(Property<T> key,T def);
public <T> T delProperty(Property<T> key);
public Config importSchema(boolean clear,Property<?> ...p);
public Property<?>[] getSchema();
}
src/main/java/com/reliancy/jabba/FileConfig.java
+123 -4
View File
@@ -7,7 +7,14 @@ You may not use this file except in compliance with the License.
*/
package com.reliancy.jabba;
import java.io.BufferedReader;
import java.io.FileReader;
import java.io.IOException;
import java.lang.reflect.Field;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import com.reliancy.util.Handy;
public class FileConfig extends Config.Base{
final Config parent;
@@ -31,13 +38,116 @@ public class FileConfig extends Config.Base{
public Config load() throws IOException{
if(parent!=null) parent.load();
if(props.isEmpty()==false) return this; // not gona load again if loaded
if(path.endsWith(".ini")){
// load ini
try(BufferedReader reader= new BufferedReader(new FileReader(path))) {
String header=null;
for(String line = reader.readLine();line!=null;line=reader.readLine()){
line=line.trim();
if(line.isEmpty() || line.startsWith("#")) continue;
if(line.startsWith("[") && line.endsWith("]")){
// this is a header
header=line.substring(1,line.length()-1);
}else{
String[] kv=Handy.split("=", line,1);
String key=kv[0];
String val=kv[1];
if(header!=null) key=header+"_"+key;
Property<?> prop=findPropertyDef(key);
if(prop!=null){
}else if("true".equalsIgnoreCase(val) || "false".equalsIgnoreCase(val)){
prop=new Property<>(key,Boolean.class);
}else if(Handy.isNumeric(val)){
prop=new Property<>(key,Float.class);
}else{
prop=new Property<>(key,String.class);
}
prop.setString(this, val);
prop.setPersistent(true);
}
}
}
}
// now do some evaluations
boolean changing=false;
int iterations=0; // to prevent recursion
do{
iterations+=1;
changing=false;
for(Property<?> p:this){
Object val=p.get(this);
if(!(val instanceof String)) continue; // skip not a string
String sval=String.valueOf(val);
if(!sval.contains("${")) continue; // no variables used
Pattern pat = java.util.regex.Pattern.compile("\\$\\{(.+?)\\}");
Matcher mat = pat.matcher(sval);
while(mat.find()){ // iterateo over matches inject other properties
Property<?> pp=findProperty(mat.group(1));
if(pp==null) continue;
sval=sval.replace(mat.group(0),pp.getString(this));
changing=true;
}
if(changing) p.setString(this,sval);
}
}while(changing && iterations<7);
return this;
}
@Override
public Config save() throws IOException{
return this;
}
protected Property<?> findProperty(String name){
Config cur=this;
while(cur!=null){
for(Property<?> p:cur){
if(name.equalsIgnoreCase(p.getName())) return p;
}
cur=cur.getParent();
}
return null;
}
/** tries to locate properties in schema and static vars. */
protected Property<?> findPropertyStat(String name,Object e){
if(e==null) return null;
if(!(e instanceof Class)) e=e.getClass();
Class<?> c=(Class<?>)e;
Field[] fields = c.getDeclaredFields();
for (Field field : fields) {
try {
if (Property.class.isAssignableFrom(field.getType())) {
Property<?> p=(Property<?>)field.get(c);
if(name.equalsIgnoreCase(p.getName())) return p;
}
}
catch (IllegalAccessException xe) {
// Handle exception here
}
}
for(Class<?> cint:c.getInterfaces()){
Property<?> p=findPropertyStat(name,cint);
if(p!=null) return p;
}
return findPropertyStat(name,c.getSuperclass());
}
protected Property<?> findPropertyDef(String name,Object...ext){
// first search over ext (objects and classes and properties)
for(Object e:ext){
if(e instanceof Property) if(((Property<?>)e).getName().equalsIgnoreCase(name)) return (Property<?>)e;
Property<?> s=findPropertyStat(name,e);
if(s!=null) return s;
}
Config cur=this;
while(cur!=null){
// lookup schema
Property<?>[] sch=cur.getSchema();
for(Property<?> p:sch) if(p.getName().equalsIgnoreCase(name)) return p;
// lookup static
Property<?> s=findPropertyStat(name,cur);
if(s!=null) return s;
cur=cur.getParent();
}
return null;
}
public FileConfig setId(String path) {
this.path = path;
return this;
@@ -53,17 +163,26 @@ public class FileConfig extends Config.Base{
}
}
/**
* FileConfig defers to parent first (it could be argsconfig) then returns own.
* if name is uppercase defers to parent first (it could be argsconfig) then returns own.
* else tries own first then checks parent.
*/
@Override
public <T> T getProperty(Config.Property<T> key, T def) {
String key_name=key.getName();
if(key_name.equals(key_name.toUpperCase())){
if(parent!=null && parent.hasProperty(key)){
return parent.getProperty(key, def);
}else if(props.containsKey(key)){
return key.getTyp().cast(props.get(key));
}else{
return def;
}
}else{
if(props.containsKey(key)){
return key.getTyp().cast(props.get(key));
}else if(parent!=null && parent.hasProperty(key)){
return parent.getProperty(key, def);
}
}
return def;
}
src/main/java/com/reliancy/jabba/FileServer.java
+37 -4
View File
@@ -6,12 +6,15 @@ You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en
You may not use this file except in compliance with the License.
*/
package com.reliancy.jabba;
import com.reliancy.util.Handy;
import com.reliancy.util.LRUCache;
import com.reliancy.util.Resources;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.URL;
import java.net.URLConnection;
import java.util.ArrayList;
import java.util.Iterator;
import org.slf4j.Logger;
@@ -21,12 +24,14 @@ import org.slf4j.Logger;
* TODO: putting, posting and maybe full DAV.
* TODO: We will need proper security.
* TODO: We will also add in memory serving.
* We have added cache control and etag support.
* Please note Router is for routing.
* Bucket is there to process input/output given verbs over resources under it.
*/
public class FileServer extends EndPoint implements AppModule,Resources.PathRewrite{
/** Bucket interface to abstract i/o and provide easier extensibility.
* asContainer matches path and then returns local-to-packet path.
* signature returns a hash over lastModified or content that reflects modification.
*/
public static interface Bucket{
String getPrefix();
@@ -34,11 +39,14 @@ public class FileServer extends EndPoint implements AppModule,Resources.PathRewr
boolean equals(String pref);
InputStream openSource(String local_path,FileServer user) throws IOException;
OutputStream openSink(String local_path,FileServer user) throws IOException;
String signature(String local_path);
}
public static class FileBucket implements Bucket{
final String prefix;
String[] extAllowed;
Object[] domain;
LRUCache<String,Long> hit_history=new LRUCache<>(2*Runtime.getRuntime().availableProcessors());
public FileBucket(String prefix){
this.prefix=prefix;
extAllowed=new String[]{};
@@ -77,11 +85,19 @@ public class FileServer extends EndPoint implements AppModule,Resources.PathRewr
Object[] sp=getDomain();
URL f=Resources.findFirst(user,local_path, sp);
if(f==null) return null; // skip if rpath not located
return f.openStream();
URLConnection conn=f.openConnection();
hit_history.put(local_path,conn.getLastModified()); // pull last modified for signature
return conn.getInputStream();
}
public OutputStream openSink(String local_path,FileServer user) throws IOException{
return null;
}
public String signature(String local_path){
Long last_modified=hit_history.get(local_path);
if(last_modified==null) return null;
String sig=String.valueOf(last_modified);
return Handy.hashMD5(sig);
}
}
final ArrayList<Bucket> buckets=new ArrayList<>();
String diskPrefix; // will be prefixed to source if file
@@ -120,26 +136,43 @@ public class FileServer extends EndPoint implements AppModule,Resources.PathRewr
}
@Override
public void serve(Request request, Response response) throws IOException {
String verb=request.getVerb();
String path=request.getPath();
Logger logger=log();
boolean atDebug=logger.isDebugEnabled();
if(atDebug) logger.debug("to serve:"+path);
if(atDebug) logger.debug("{0}:{1}",verb,path);
for(Bucket bucket:buckets){
String local_path=bucket.asContained(path);
if(local_path==null) continue; // this bucket is not accepting
if(HTTP.VERB_GET.equals(verb)){
try(InputStream ins=bucket.openSource(local_path,this)){
if(ins==null) continue; // url did not take
String etag=bucket.signature(local_path);
if(etag!=null){
response.setHeader("Cache-Control","max-age=0, must-revalidate");
response.setHeader("ETag",etag);
String etag_old=request.getHeader("If-None-Match");
if(etag.equals(etag_old)){
// we got same etag no change
response.setStatus(Response.HTTP_NOT_MODIFIED);
return;
}
}
if(atDebug) logger.debug("\tfound:"+local_path);
String ctype=HTTP.ext2mime(local_path);
response.setStatus(Response.HTTP_OK);
response.setContentType(ctype);
ResponseEncoder enc=response.getEncoder();
enc.writeStream(ins);
return;
return; // we got something
}
}else{
// these verbs are not supported
}
}
response.setStatus(Response.HTTP_NOT_FOUND);
response.getEncoder().writeln("missing file:{0}",path);
logger.error("not found:"+path);
logger.error("not found:{0}",path);
}
/**
* Will render a URL resource to response.
src/main/java/com/reliancy/jabba/HTTP.java
+6
View File
@@ -15,6 +15,12 @@ import java.util.HashMap;
*
*/
public final class HTTP {
public static String VERB_GET="GET";
public static String VERB_PUT="PUT";
public static String VERB_DEL="DELETE";
public static String VERB_POST="POST";
public static String VERB_HEAD="HEAD";
public static String MIME_PLAIN="text/plain";
public static String MIME_JSON="application/json";
public static String MIME_BYTES="application/octet-stream";
src/main/java/com/reliancy/jabba/MethodEndPoint.java
+1 -1
View File
@@ -66,7 +66,7 @@ public class MethodEndPoint extends EndPoint{
}
@Override
public void serve(Request request, Response response) throws IOException{
log().info("Serving method....{}",invokeType);
log().debug("Serving method....{}",invokeType);
try{
Object ret=null;
switch(invokeType){
src/main/java/com/reliancy/jabba/Response.java
+1
View File
@@ -33,6 +33,7 @@ public class Response {
public static final int HTTP_FORBIDDEN=HttpServletResponse.SC_FORBIDDEN;
public static final int HTTP_TEMPORARY_REDIRECT=HttpServletResponse.SC_TEMPORARY_REDIRECT;
public static final int HTTP_FOUND_REDIRECT=HttpServletResponse.SC_FOUND;
public static final int HTTP_NOT_MODIFIED=HttpServletResponse.SC_NOT_MODIFIED;
final protected HttpServletResponse http_response;
final protected Writer char_response;
src/main/java/com/reliancy/util/Handy.java
+3
View File
@@ -120,6 +120,9 @@ public final class Handy {
if( Float.class==( clazz ) || float.class==( clazz ) ) return Float.parseFloat( value );
if( Double.class==( clazz ) || double.class==( clazz )) return Double.parseDouble( value );
}
if(clazz==String.class || clazz==CharSequence.class){
return String.valueOf(val);
}
return val;
}
/**
src/main/java/com/reliancy/util/LRUCache.java
+79
View File
@@ -0,0 +1,79 @@
package com.reliancy.util;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.Map;
/** Least recently used cache is a useful map of sorts.
* It has a fixed capacity and it forgets least used entries if new are added.
* If an allocator is installed it is consulted on cache miss.
* If a disposer is installed is is consulted on cache overflow.
* We can provide the same object that implements both and make use of a pool.
*/
public class LRUCache<K,V>{
public static interface Allocator<K,V>{
V request(K key);
}
public static interface Disposer<K,V>{
void release(K key,V val);
}
final Map<K,V> data;
int capacity;
final LinkedList<K> order=new LinkedList<>();
Allocator<K,V> allocator;
Disposer<K,V> disposer;
public LRUCache(int capacity,Map<K,V> backend){
this.capacity=capacity;
data=backend!=null?backend:new HashMap<K,V>();
}
public LRUCache(int capacity){
this(capacity,null);
}
public LRUCache<K,V> setAllocator(Allocator<K,V> a){
allocator=a;
return this;
}
public LRUCache<K,V> setDisposer(Disposer<K,V> a){
disposer=a;
return this;
}
public int size() {
return data.size();
}
public boolean containsKey(Object key) {
return data.containsKey(key);
}
public boolean containsValue(Object value) {
return data.containsValue(value);
}
public V get(K key) {
V ret=data.get(key);
if(ret!=null){
//cache is hit
order.remove(key);
order.addFirst(key);
}else{
//cache is missed
ret=allocator!=null?allocator.request(key):null;
}
return ret;
}
public V put(K key, V value) {
if(order.size()>=capacity){
// capacity is reached
K last=order.removeLast();
data.remove(last);
if(disposer!=null) disposer.release(key, value);
}
order.addFirst(key);
return data.put(key,value);
}
public V remove(Object key) {
order.remove(key);
return data.remove(key);
}
public void clear() {
order.clear();
data.clear();
}
}
src/main/java/com/reliancy/util/Resources.java
+35 -8
View File
@@ -30,11 +30,15 @@ import java.nio.charset.StandardCharsets;
* such as Template or FileServe (unless overriden.)
*/
public class Resources {
public static interface PathRewrite{
public String rewritePath(String path,Object context);
}
public static Object[] search_path;
/** appends one+ paths to search at position pos.
* neg pos substracts from end
*/
public static Object[] appendSearch(int pos,Object ...src){
//search_history.clear();
if(search_path==null) search_path=new Object[0];
if(pos<0) pos=search_path.length+pos+1;
if(pos<0 || pos>search_path.length) throw new IndexOutOfBoundsException("at:"+pos);
@@ -52,9 +56,14 @@ public class Resources {
search_path=new_path;
return search_path;
}
public static interface PathRewrite{
public String rewritePath(String path,Object context);
}
/** returns first good URL for path over sp or search_path.
* Along the way it optionally rewrites path to adjust to search path context.
* When possible it records lastModified timestamp in search_history for later lookup.
* @param remap rewrite rule if any
* @param path path to locate
* @param sp search path reverts to search_path if not specified
* @return URL that can be read.
*/
public static URL findFirst(PathRewrite remap,String path,Object ... sp){
String path0=path;
if(sp==null || sp.length==0) sp=search_path;
@@ -67,7 +76,9 @@ public class Resources {
File ff=new File(base.toString(),path);
if(ff.exists()){
try {
return ff.toURI().toURL();
URL ret=ff.toURI().toURL();
//search_history.put(path0,ff.lastModified());
return ret;
} catch (MalformedURLException e) {
continue;
}
@@ -76,7 +87,9 @@ public class Resources {
File ff=new File((File)base,path);
if(ff.exists()){
try {
return ff.toURI().toURL();
URL ret=ff.toURI().toURL();
//search_history.put(path,ff.lastModified());
return ret;
} catch (MalformedURLException e) {
continue;
}
@@ -91,7 +104,10 @@ public class Resources {
huc=(HttpURLConnection) ret.openConnection();
huc.setRequestMethod("HEAD");
int responseCode = huc.getResponseCode();
if(responseCode==HttpURLConnection.HTTP_OK) return ret;
if(responseCode==HttpURLConnection.HTTP_OK){
//search_history.put(path,huc.getLastModified());
return ret;
}
}finally{
if(huc!=null) huc.disconnect();
}
@@ -99,11 +115,17 @@ public class Resources {
if(proto.startsWith("jar")){
JarURLConnection juc = null;
juc=(JarURLConnection) ret.openConnection();
if(juc.getJarEntry()!=null) return ret;
if(juc.getJarEntry()!=null){
//search_history.put(path,juc.getLastModified());
return ret;
}
}
if(proto.equals("file")){
File f=new File(ret.getPath());
if(f.exists()) return ret;
if(f.exists()){
//search_history.put(path,f.lastModified());
return ret;
}
}
} catch (MalformedURLException e) {
continue;
@@ -114,6 +136,11 @@ public class Resources {
}
return null;
}
/** if recorded in previous searches returns time modified. */
// public static Long lastModified(String path){
// Long ret=search_history.get(path);
// return ret;
// }
public static String toString(URL url) throws IOException{
return toString(url,StandardCharsets.UTF_8);
}
src/test/java/com/reliancy/jabba/FileConfigTest.java
+25
View File
@@ -0,0 +1,25 @@
package com.reliancy.jabba;
import static org.junit.Assert.assertTrue;
import java.io.IOException;
import org.junit.Test;
public class FileConfigTest {
@Test
public void testFile(){
System.out.println("testing file config...");
ArgsConfig args0=null;//new ArgsConfig();
FileConfig args=new FileConfig(args0,"./var/conf.ini");
try {
args.load();
for(ArgsConfig.Property<?> p:args){
System.out.println("p:"+p+"="+p.get(args));
}
} catch (IOException e) {
e.printStackTrace();
assertTrue(false);
}
}
}