Skip to content

CHANGED: .xml added as default extension to ignore as this can interfere with sitemaps #39

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
mgoodfellow wants to merge 1 commit into
base: master
Choose a base branch
from
Open

CHANGED: .xml added as default extension to ignore as this can interfere with sitemaps #39

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 15 additions & 0 deletions .editorconfig
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
# editorconfig.org
root = true

[*]
indent_style = space
end_of_line = crlf
charset = utf-8
trim_trailing_whitespace = true
insert_final_newline = true

[**/*.{config,csproj,dotsettings,feature,fxcop,js,json,markdown,md,ncrunch*,ndepend,rb,targets,xml,xsd}]
indent_size = 2

[**/*.cs]
indent_size = 4
26 changes: 13 additions & 13 deletions Prerender.io/PrerenderConfigSection.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
using System;
using System;
using System.Collections.Generic;
using System.Configuration;

Expand All @@ -20,18 +20,18 @@ public String PrerenderServiceUrl
}
}

[ConfigurationProperty("stripApplicationNameFromRequestUrl", DefaultValue = false)]
public bool StripApplicationNameFromRequestUrl
{
get
{
return (bool)this["stripApplicationNameFromRequestUrl"];
}
set
{
this["stripApplicationNameFromRequestUrl"] = value;
}
}
[ConfigurationProperty("stripApplicationNameFromRequestUrl", DefaultValue = false)]
public bool StripApplicationNameFromRequestUrl
{
get
{
return (bool)this["stripApplicationNameFromRequestUrl"];
}
set
{
this["stripApplicationNameFromRequestUrl"] = value;
}
}

[ConfigurationProperty("whitelist")]
public String WhitelistString
Expand Down
33 changes: 15 additions & 18 deletions Prerender.io/PrerenderModule.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
using System;
using System;
using System.Collections.Generic;
using System.Configuration;
using System.Diagnostics;
Expand Down Expand Up @@ -139,15 +139,15 @@ private String GetApiUrl(HttpRequest request)
}

// Remove the application from the URL
if (_prerenderConfig.StripApplicationNameFromRequestUrl && !string.IsNullOrEmpty(request.ApplicationPath) && request.ApplicationPath != "/")
{
// http://test.com/MyApp/?_escape_=/somewhere
url = url.Replace(request.ApplicationPath, string.Empty);
}
if (_prerenderConfig.StripApplicationNameFromRequestUrl && !string.IsNullOrEmpty(request.ApplicationPath) && request.ApplicationPath != "/")
{
// http://test.com/MyApp/?_escape_=/somewhere
url = url.Replace(request.ApplicationPath, string.Empty);
}

var prerenderServiceUrl = _prerenderConfig.PrerenderServiceUrl;
return prerenderServiceUrl.EndsWith("/")
? (prerenderServiceUrl + url)
? prerenderServiceUrl + url
: string.Format("{0}/{1}", prerenderServiceUrl, url);
}

Expand All @@ -162,7 +162,7 @@ public static string RemoveQueryStringByKey(string url, string key)
newQueryString.Remove(key);

// this gets the page path from root without QueryString
string pagePathWithoutQueryString = uri.GetLeftPart(UriPartial.Path);
var pagePathWithoutQueryString = uri.GetLeftPart(UriPartial.Path);

return newQueryString.Count > 0
? String.Format("{0}?{1}", pagePathWithoutQueryString, newQueryString)
Expand All @@ -178,10 +178,6 @@ private bool ShouldShowPrerenderedPage(HttpRequest request)

var blacklist = _prerenderConfig.Blacklist;
if (blacklist != null && IsInBlackList(url, referer, blacklist))




{
return false;
}
Expand All @@ -205,15 +201,16 @@ private bool ShouldShowPrerenderedPage(HttpRequest request)
{
return false;
}

if (IsInResources(url))
{
return false;
}
return true;

return true;
}

private bool IsInBlackList(Uri url, string referer, IEnumerable<string> blacklist)
private static bool IsInBlackList(Uri url, string referer, IEnumerable<string> blacklist)
{
return blacklist.Any(item =>
{
Expand All @@ -222,7 +219,7 @@ private bool IsInBlackList(Uri url, string referer, IEnumerable<string> blacklis
});
}

private bool IsInWhiteList(Uri url, IEnumerable<string> whiteList)
private static bool IsInWhiteList(Uri url, IEnumerable<string> whiteList)
{
return whiteList.Any(item => new Regex(item).IsMatch(url.AbsoluteUri));
}
Expand All @@ -238,7 +235,7 @@ private IEnumerable<String> GetExtensionsToIgnore()
var extensionsToIgnore = new List<string>(new[]{".js", ".css", ".less", ".png", ".jpg", ".jpeg",
".gif", ".pdf", ".doc", ".txt", ".zip", ".mp3", ".rar", ".exe", ".wmv", ".doc", ".avi", ".ppt", ".mpg",
".mpeg", ".tif", ".wav", ".mov", ".psd", ".ai", ".xls", ".mp4", ".m4a", ".swf", ".dat", ".dmg",
".iso", ".flv", ".m4v", ".torrent"});
".iso", ".flv", ".m4v", ".torrent", ".xml"});
if (_prerenderConfig.ExtensionsToIgnore.IsNotEmpty())
{
extensionsToIgnore.AddRange(_prerenderConfig.ExtensionsToIgnore);
Expand Down Expand Up @@ -276,7 +273,7 @@ private IEnumerable<String> GetCrawlerUserAgents()
return crawlerUserAgents;
}

private bool HasEscapedFragment(HttpRequest request)
private static bool HasEscapedFragment(HttpRequest request)
{
return request.QueryString.AllKeys.Contains(_Escaped_Fragment);
}
Expand Down