Sitemaps XML format https://www.sitemaps.org/protocol.html
1.robots.txt
step 1: Controller
\src\Presentation\Nop.Web\Controllers\CommonController.cs
public virtual IActionResult RobotsTextFile()
{
var robotsFileContent = _commonModelFactory.PrepareRobotsTextFile();
return Content(robotsFileContent, MimeTypes.TextPlain);
}src\Presentation\Nop.Web\Factories\CommonModelFactory.cs
if robots.custom.txt exists return robots.custom.txt
if robots.custom.txt not exists create and return robots.txt
if robots.custom.txt not exists but robots.additions.txt exists，first create robots.txt ，then add robots.custom.txt and return robots.txt
public virtual string PrepareRobotsTextFile()
{
var sb = new StringBuilder();
//if robots.custom.txt exists, let's use it instead of hard-coded data below
var robotsFilePath = _fileProvider.Combine(_fileProvider.MapPath("~/"), "robots.custom.txt");
//if robots.custom.txt exists, let's use it instead of hard-coded data below
// string robotsFilePath = System.IO.Path.Combine(CommonHelper.MapPath("~/"), "robots.custom.txt");
if (System.IO.File.Exists(robotsFilePath))
{
//the robots.txt file exists
string robotsFileContent = System.IO.File.ReadAllText(robotsFilePath);
sb.Append(robotsFileContent);
}
else
{
//doesn't exist. Let's generate it (default behavior)
var disallowPaths = new List<string>
{
"/admin",
"/bin/",
"/files/",
"/files/exportimport/",
"/country/getstatesbycountryid",
"/install",
"/setproductreviewhelpfulness",
};
var localizableDisallowPaths = new List<string>
{
"/addproducttocart/catalog/",
"/addproducttocart/details/",
"/wishlist",
};
const string newLine = "\r\n"; //Environment.NewLine
sb.Append("User-agent: *");
sb.Append(newLine);
//sitemaps
if (_localizationSettings.SeoFriendlyUrlsForLanguagesEnabled)
{
//URLs are localizable. Append SEO code
foreach (var language in _languageService.GetAllLanguages(storeId: _storeContext.CurrentStore.Id.ToString()))
{
sb.AppendFormat("Sitemap: {0}{1}/sitemap.xml", _storeContext.CurrentStore.Url, language.UniqueSeoCode);
sb.Append(newLine);
}
}
else
{
//localizable paths (without SEO code)
sb.AppendFormat("Sitemap: {0}sitemap.xml", _storeContext.CurrentStore.Url);
sb.Append(newLine);
}
//usual paths
foreach (var path in disallowPaths)
{
sb.AppendFormat("Disallow: {0}", path);
sb.Append(newLine);
}
//localizable paths (without SEO code)
foreach (var path in localizableDisallowPaths)
{
sb.AppendFormat("Disallow: {0}", path);
sb.Append(newLine);
}
if (_localizationSettings.SeoFriendlyUrlsForLanguagesEnabled)
{
//URLs are localizable. Append SEO code
foreach (var language in _languageService.GetAllLanguages(storeId: _storeContext.CurrentStore.Id.ToString()))
{
foreach (var path in localizableDisallowPaths)
{
sb.AppendFormat("Disallow: /{0}{1}", language.UniqueSeoCode, path);
sb.Append(newLine);
}
}
}
//load and add robots.txt additions to the end of file.
string robotsAdditionsFile = System.IO.Path.Combine(CommonHelper.MapPath("~/"), "robots.additions.txt");
if (System.IO.File.Exists(robotsAdditionsFile))
{
string robotsFileContent = System.IO.File.ReadAllText(robotsAdditionsFile);
sb.Append(robotsFileContent);
}
}
return sb.ToString();
}
http://localhost:6999/common/RobotsTextFile
User-agent: *
Allow: /
User-agent: MJ12bot
Disallow: /step 2: register router
http://localhost:6999/Robots.txt
User-agent: *
Allow: /
2.sitemap.xml
step
1.get all url come from product，blog etc. to list<url>
2.convert list<url> to var sitemapUrls = new List<SitemapUrl>();
3. write sitemap.xml come from List<SitemapUrl>()
4.CommonController's action SitemapXml() output sitemap.xml as “Text/XML”
5.register router,which convert CommonController ,action SitemapXml() to host://sitemap.xml
\src\Presentation\Nop.Web\Controllers\CommonController.cs
[CheckAccessClosedStore(true)]
public virtual IActionResult SitemapXml(int? id)
{
if (!_commonSettings.SitemapEnabled)
return Content("_commonSettings.SitemapEnabled is Disable");
// return RedirectToRoute("HomePage");
var siteMap = _commonModelFactory.PrepareSitemapXml(this.Url, id);
return Content(siteMap, "text/xml");
}\src\Presentation\Nop.Web\Factories\CommonModelFactory.cs
public virtual string PrepareSitemapXml(IUrlHelper url, int? id)
{
string cacheKey = string.Format(ModelCacheEventConsumer.SITEMAP_SEO_MODEL_KEY, id,
_workContext.WorkingLanguage.Id,
string.Join(",", _workContext.CurrentCustomer.GetCustomerRoleIds()),
_storeContext.CurrentStore.Id);
var siteMap = _cacheManager.Get(cacheKey, () => _sitemapGenerator.Generate(url, id));
return siteMap;
}This will build an XML sitemap for better index with search engines.page or not.
/// <summary>
/// This will build an XML sitemap for better index with search engines.
/// See http://en.wikipedia.org/wiki/Sitemaps for more information.
/// </summary>
/// <param name="urlHelper">URL helper</param>
/// <param name="id">Sitemap identifier</param>
/// <returns>Sitemap.xml as string</returns>
public virtual string Generate(IUrlHelper urlHelper, int? id)
{
using (var stream = new MemoryStream())
{
Generate(urlHelper, stream, id);
return Encoding.UTF8.GetString(stream.ToArray());
}
}
/// <summary>
/// This will build an XML sitemap for better index with search engines.
/// See http://en.wikipedia.org/wiki/Sitemaps for more information.
/// </summary>
/// <param name="urlHelper">URL helper</param>
/// <param name="id">Sitemap identifier</param>
/// <param name="stream">Stream of sitemap.</param>
public virtual void Generate(IUrlHelper urlHelper, Stream stream, int? id)
{
//generate all URLs for the sitemap
var sitemapUrls = GenerateUrls(urlHelper);
//split URLs into separate lists based on the max size
var sitemaps = sitemapUrls.Select((url, index) => new { Index = index, Value = url })
.GroupBy(group => group.Index / maxSitemapUrlNumber).Select(group => group.Select(url => url.Value).ToList()).ToList();
if (!sitemaps.Any())
return;
if (id.HasValue)
{
//requested sitemap does not exist
if (id.Value == 0 || id.Value > sitemaps.Count)
return;
//otherwise write a certain numbered sitemap file into the stream
WriteSitemap(urlHelper, stream, sitemaps.ElementAt(id.Value - 1));
}
else
{
//URLs more than the maximum allowable, so generate a sitemap index file
if (sitemapUrls.Count >= maxSitemapUrlNumber)
{
//write a sitemap index file into the stream
WriteSitemapIndex(urlHelper, stream, sitemaps.Count);
}
else
{
//otherwise generate a standard sitemap
WriteSitemap(urlHelper, stream, sitemaps.First());
}
}
}
Add all url to IList<SitemapUrl>
protected virtual IList<SitemapUrl> GenerateUrls(IUrlHelper urlHelper)
{
var sitemapUrls = new List<SitemapUrl>();
//home page
var homePageUrl = urlHelper.RouteUrl("HomePage", null, GetHttpProtocol());
sitemapUrls.Add(new SitemapUrl(homePageUrl, UpdateFrequency.Weekly, DateTime.UtcNow));
//search products
var productSearchUrl = urlHelper.RouteUrl("ProductSearch", null, GetHttpProtocol());
sitemapUrls.Add(new SitemapUrl(productSearchUrl, UpdateFrequency.Weekly, DateTime.UtcNow));
//contact us
var contactUsUrl = urlHelper.RouteUrl("ContactUs", null, GetHttpProtocol());
sitemapUrls.Add(new SitemapUrl(contactUsUrl, UpdateFrequency.Weekly, DateTime.UtcNow));
//topics
sitemapUrls.AddRange(GetTopicUrls(urlHelper));
//blog posts
sitemapUrls.AddRange( GetBlogPostsUrls(urlHelper).Result);
...
return sitemapUrls;
}
step 2: register router
\src\Presentation\Nop.Web\Infrastructure\RouteProvider.cs
public static class RegisterRoutes
{
/// <summary>
/// static router table
/// </summary>
/// <param name="routes"></param>
public static void MapModuleRoute(this IRouteBuilder routes)
{
//sitemap (XML)
routes.MapRoute("sitemap.xml", "sitemap.xml",
new { controller = "Common", action = "SitemapXml" });
routes.MapRoute("sitemap-indexed.xml", "sitemap-{Id:min(0)}.xml",
new { controller = "Common", action = "SitemapXml" });
//robots.txt
routes.MapRoute("robots.txt", "robots.txt",
new { controller = "Common", action = "RobotsTextFile" });
}
}
public void Configure(IApplicationBuilder app, IHostingEnvironment env, IApplicationLifetime appLifetime, ILoggerFactory loggerFactory)
{
routes.MapModuleRoute();
}